655 lines
24 KiB
Python
655 lines
24 KiB
Python
import pandas as pd
|
||
import numpy as np
|
||
from sklearn.linear_model import Ridge
|
||
from sklearn.model_selection import GridSearchCV
|
||
from sklearn.preprocessing import StandardScaler
|
||
from sklearn.metrics import r2_score, mean_squared_error
|
||
import joblib
|
||
import os
|
||
from datetime import datetime
|
||
from dateutil.relativedelta import relativedelta
|
||
|
||
|
||
class RidgePredictor:
|
||
def __init__(self, base_model_path='models'):
|
||
"""
|
||
初始化预测器
|
||
base_model_path: 模型存储的基础路径
|
||
"""
|
||
self.base_model_path = base_model_path
|
||
self.scaler = StandardScaler()
|
||
os.makedirs(base_model_path, exist_ok=True)
|
||
|
||
def get_project_features(self, df, project):
|
||
"""获取项目对应的特征列名"""
|
||
try:
|
||
features = [col for col in df.columns if col.startswith(f'{project}_')]
|
||
if not features:
|
||
raise ValueError(f"No features found for project {project}")
|
||
return features
|
||
except Exception as e:
|
||
print(f"Error in get_project_features: {str(e)}")
|
||
raise
|
||
|
||
def print_ridge_formula(self, model, feature_names, scaler):
|
||
"""打印岭回归公式"""
|
||
try:
|
||
coefficients = model.coef_
|
||
intercept = model.intercept_
|
||
|
||
formula = "y = "
|
||
for i, (coef, name) in enumerate(zip(coefficients, feature_names)):
|
||
if i > 0:
|
||
formula += " + " if coef >= 0 else " - "
|
||
else:
|
||
formula += "-" if coef < 0 else ""
|
||
formula += f"{abs(coef):.4f} * {name}"
|
||
|
||
formula += f" + {intercept:.4f}"
|
||
|
||
# 添加标准化说明
|
||
formula += "注：特征已进行标准化处理，使用以下参数："
|
||
for i, name in enumerate(feature_names):
|
||
formula += f"{name}: mean={scaler.mean_[i]:.4f}, scale={scaler.scale_[i]:.4f}"
|
||
return formula
|
||
except Exception as e:
|
||
print(f"Error in print_ridge_formula: {str(e)}")
|
||
raise
|
||
|
||
def get_feature_average(self, df, area, project, year, month, n_months=3):
|
||
"""获取前n个月的特征平均值"""
|
||
try:
|
||
feature_cols = self.get_project_features(df, project)
|
||
|
||
# 计算前n个月的日期范围
|
||
dates = []
|
||
curr_year, curr_month = year, month
|
||
for _ in range(n_months):
|
||
curr_month -= 1
|
||
if curr_month == 0:
|
||
curr_month = 12
|
||
curr_year -= 1
|
||
dates.append((curr_year, curr_month))
|
||
|
||
# 获取特征值
|
||
features_list = []
|
||
for year, month in dates:
|
||
mask = (df['area_name'] == area) & (df['year'] == year) & (df['month'] == month)
|
||
if sum(mask) > 0:
|
||
features_list.append(df[mask][feature_cols].values[0])
|
||
|
||
if not features_list:
|
||
raise ValueError(f"No feature data found for {area} in specified months")
|
||
|
||
return np.mean(features_list, axis=0)
|
||
except Exception as e:
|
||
print(f"Error in get_feature_average: {str(e)}")
|
||
raise
|
||
|
||
def train_and_evaluate(self, df, area, project, target_year, target_month):
|
||
"""训练模型并评估性能"""
|
||
try:
|
||
feature_cols = self.get_project_features(df, project)
|
||
|
||
# 准备训练数据
|
||
dates = []
|
||
curr_year, curr_month = target_year, target_month
|
||
for _ in range(12): # 使用前12个月的数据
|
||
curr_month -= 1
|
||
if curr_month == 0:
|
||
curr_month = 12
|
||
curr_year -= 1
|
||
dates.append((curr_year, curr_month))
|
||
|
||
train_data = []
|
||
for year, month in dates:
|
||
mask = (df['area_name'] == area) & (df['year'] == year) & (df['month'] == month)
|
||
if sum(mask) > 0:
|
||
train_data.append(df[mask])
|
||
|
||
if not train_data:
|
||
raise ValueError(f"No training data found for {area}")
|
||
|
||
train_df = pd.concat(train_data)
|
||
|
||
X = train_df[feature_cols].values
|
||
y = train_df[project].values
|
||
|
||
# 数据标准化
|
||
X_scaled = self.scaler.fit_transform(X)
|
||
|
||
# 网格搜索最佳参数
|
||
param_grid = {'alpha': np.logspace(-3, 3, 7)}
|
||
ridge = Ridge()
|
||
grid_search = GridSearchCV(ridge, param_grid, cv=5, scoring='neg_mean_squared_error')
|
||
grid_search.fit(X_scaled, y)
|
||
|
||
# 获取最佳模型
|
||
best_model = grid_search.best_estimator_
|
||
|
||
# 计算评估指标
|
||
y_pred = best_model.predict(X_scaled)
|
||
r2 = r2_score(y, y_pred)
|
||
mse = mean_squared_error(y, y_pred)
|
||
|
||
# 保存模型和标准化器
|
||
model_dir = os.path.join(self.base_model_path, f"{area}_{project}")
|
||
os.makedirs(model_dir, exist_ok=True)
|
||
|
||
model_path = os.path.join(model_dir, f"model_{target_year}_{target_month}.pkl")
|
||
scaler_path = os.path.join(model_dir, f"scaler_{target_year}_{target_month}.pkl")
|
||
|
||
joblib.dump(best_model, model_path)
|
||
joblib.dump(self.scaler, scaler_path)
|
||
|
||
# 生成岭回归公式
|
||
formula = self.print_ridge_formula(best_model, feature_cols, self.scaler)
|
||
|
||
return best_model, self.scaler, r2, mse, formula
|
||
|
||
except Exception as e:
|
||
print(f"Error in train_and_evaluate: {str(e)}")
|
||
raise
|
||
|
||
def get_feature_average(self, df, area, project, year, month, n_months=3):
|
||
"""获取前n个月的特征平均值"""
|
||
try:
|
||
feature_cols = self.get_project_features(df, project)
|
||
|
||
# 计算前n个月的日期范围
|
||
dates = []
|
||
curr_year, curr_month = year, month
|
||
for _ in range(n_months):
|
||
curr_month -= 1
|
||
if curr_month == 0:
|
||
curr_month = 12
|
||
curr_year -= 1
|
||
dates.append((curr_year, curr_month))
|
||
|
||
# 获取特征值
|
||
features_list = []
|
||
for year, month in dates:
|
||
mask = (df['area_name'] == area) & (df['year'] == year) & (df['month'] == month)
|
||
if sum(mask) > 0:
|
||
features_list.append(df[mask][feature_cols].values[0])
|
||
|
||
if not features_list:
|
||
raise ValueError(f"No feature data found for {area} in specified months")
|
||
|
||
return np.mean(features_list, axis=0)
|
||
except Exception as e:
|
||
print(f"Error in get_feature_average: {str(e)}")
|
||
raise
|
||
|
||
def train_and_evaluate(self, df, area, project, target_year, target_month):
|
||
"""训练模型并评估性能"""
|
||
try:
|
||
feature_cols = self.get_project_features(df, project)
|
||
|
||
# 准备训练数据
|
||
dates = []
|
||
curr_year, curr_month = target_year, target_month
|
||
for _ in range(12): # 使用前12个月的数据
|
||
curr_month -= 1
|
||
if curr_month == 0:
|
||
curr_month = 12
|
||
curr_year -= 1
|
||
dates.append((curr_year, curr_month))
|
||
|
||
train_data = []
|
||
for year, month in dates:
|
||
mask = (df['area_name'] == area) & (df['year'] == year) & (df['month'] == month)
|
||
if sum(mask) > 0:
|
||
train_data.append(df[mask])
|
||
|
||
if not train_data:
|
||
raise ValueError(f"No training data found for {area}")
|
||
|
||
train_df = pd.concat(train_data)
|
||
|
||
X = train_df[feature_cols].values
|
||
y = train_df[project].values
|
||
|
||
# 数据标准化
|
||
X_scaled = self.scaler.fit_transform(X)
|
||
|
||
# 网格搜索最佳参数
|
||
param_grid = {'alpha': np.logspace(-3, 3, 7)}
|
||
ridge = Ridge()
|
||
grid_search = GridSearchCV(ridge, param_grid, cv=5, scoring='neg_mean_squared_error')
|
||
grid_search.fit(X_scaled, y)
|
||
print(f"区域: {area}, 项目: {project} - 当前模型的最佳参数: {grid_search.best_params_}")
|
||
|
||
# 获取最佳模型
|
||
best_model = grid_search.best_estimator_
|
||
|
||
# 计算评估指标
|
||
y_pred = best_model.predict(X_scaled)
|
||
r2 = r2_score(y, y_pred)
|
||
mse = mean_squared_error(y, y_pred)
|
||
|
||
# 保存模型和标准化器
|
||
model_dir = os.path.join(self.base_model_path, f"{area}_{project}")
|
||
os.makedirs(model_dir, exist_ok=True)
|
||
|
||
model_path = os.path.join(model_dir, f"model_{target_year}_{target_month}.pkl")
|
||
scaler_path = os.path.join(model_dir, f"scaler_{target_year}_{target_month}.pkl")
|
||
|
||
joblib.dump(best_model, model_path)
|
||
joblib.dump(self.scaler, scaler_path)
|
||
|
||
# 生成岭回归公式
|
||
formula = self.print_ridge_formula(best_model, feature_cols, self.scaler)
|
||
|
||
return best_model, self.scaler, r2, mse, formula
|
||
|
||
except Exception as e:
|
||
print(f"Error in train_and_evaluate: {str(e)}")
|
||
raise
|
||
|
||
def predict_with_history(self, df, area, project, start_year=2023, start_month=1, end_month=6):
|
||
"""使用历史数据进行预测（2023.1-6）"""
|
||
try:
|
||
results = []
|
||
feature_importance = []
|
||
model_metrics = []
|
||
|
||
for month in range(start_month, end_month + 1):
|
||
print(f"Predicting {area} - {project} for {start_year}/{month}")
|
||
|
||
# 训练模型并评估
|
||
model, scaler, r2, mse, formula = self.train_and_evaluate(
|
||
df, area, project, start_year, month
|
||
)
|
||
|
||
# 获取预测用的特征（前3个月平均）
|
||
test_features = self.get_feature_average(
|
||
df, area, project, start_year, month
|
||
)
|
||
|
||
# 标准化特征并预测
|
||
test_features_scaled = scaler.transform(test_features.reshape(1, -1))
|
||
prediction = model.predict(test_features_scaled)[0]
|
||
|
||
# 获取特征重要性
|
||
feature_cols = self.get_project_features(df, project)
|
||
importance = abs(model.coef_)
|
||
|
||
# 修改特征重要性的存储格式
|
||
importance_dict = {
|
||
'area': area,
|
||
'year': start_year,
|
||
'month': month,
|
||
}
|
||
# 将特征重要性存储为 project_feature: value 的格式
|
||
for feat, imp in zip(feature_cols, importance):
|
||
importance_dict[f'{feat}'] = imp
|
||
|
||
# 查找是否已存在相同area/year/month的记录
|
||
existing_idx = next((
|
||
i for i, x in enumerate(feature_importance)
|
||
if x['area'] == area and x['year'] == start_year and x['month'] == month
|
||
), None)
|
||
|
||
if existing_idx is not None:
|
||
# 如果存在，更新现有记录
|
||
feature_importance[existing_idx].update(importance_dict)
|
||
else:
|
||
# 如果不存在，添加新记录
|
||
feature_importance.append(importance_dict)
|
||
|
||
results.append({
|
||
'area': area,
|
||
'project': project,
|
||
'year': start_year,
|
||
'month': month,
|
||
'prediction': prediction,
|
||
'actual': df[(df['area_name'] == area) &
|
||
(df['year'] == start_year) &
|
||
(df['month'] == month)][project].values[0]
|
||
})
|
||
|
||
model_metrics.append({
|
||
'area': area,
|
||
'project': project,
|
||
'year': start_year,
|
||
'month': month,
|
||
'r2': r2,
|
||
'mse': mse,
|
||
'formula': formula
|
||
})
|
||
|
||
return results, feature_importance, model_metrics
|
||
|
||
except Exception as e:
|
||
print(f"Error in predict_with_history: {str(e)}")
|
||
raise
|
||
def predict_future(self, df, area, project, start_month=7, end_month=9, year=2023):
|
||
"""预测未来月份（2023.7-9）"""
|
||
try:
|
||
results = []
|
||
|
||
# 加载6月的模型
|
||
model_dir = os.path.join(self.base_model_path, f"{area}_{project}")
|
||
model = joblib.load(os.path.join(model_dir, f"model_{year}_6.pkl"))
|
||
scaler = joblib.load(os.path.join(model_dir, f"scaler_{year}_6.pkl"))
|
||
|
||
for month in range(start_month, end_month + 1):
|
||
print(f"Predicting future {area} - {project} for {year}/{month}")
|
||
|
||
if month == 7:
|
||
# 使用4,5,6月的特征平均值
|
||
test_features = np.mean([
|
||
self.get_feature_average(df, area, project, year, m)
|
||
for m in [4, 5, 6]
|
||
], axis=0)
|
||
else:
|
||
# 使用前三个预测结果的特征
|
||
test_features = np.mean([
|
||
self.get_feature_average(df, area, project, year, m)
|
||
for m in range(month - 3, month)
|
||
], axis=0)
|
||
|
||
test_features_scaled = scaler.transform(test_features.reshape(1, -1))
|
||
prediction = model.predict(test_features_scaled)[0]
|
||
|
||
results.append({
|
||
'area': area,
|
||
'project': project,
|
||
'year': year,
|
||
'month': month,
|
||
'prediction': prediction
|
||
})
|
||
|
||
return results
|
||
|
||
except Exception as e:
|
||
print(f"Error in predict_future: {str(e)}")
|
||
raise
|
||
|
||
def format_all_predictions(all_results, all_future_results):
|
||
"""
|
||
将所有预测结果格式化为指定格式
|
||
"""
|
||
# 创建一个字典来存储所有结果
|
||
formatted_data = {}
|
||
|
||
# 合并所有结果
|
||
all_predictions = all_results + all_future_results
|
||
|
||
# 处理每个预测结果
|
||
for result in all_predictions:
|
||
key = (result['area'], result['year'], result['month'])
|
||
|
||
if key not in formatted_data:
|
||
formatted_data[key] = {
|
||
'area_name': result['area'],
|
||
'year': result['year'],
|
||
'month': result['month'],
|
||
'xxzs': 0,
|
||
'zfjc': 0,
|
||
'tsjb': 0,
|
||
'xzcf': 0,
|
||
'cckh': 0,
|
||
'cjjc': 0,
|
||
'zhxzb': 0
|
||
}
|
||
|
||
# 将预测值转换为百分比并保存
|
||
project = result['project']
|
||
formatted_data[key][project] = round(result['prediction'] * 100, 8)
|
||
|
||
# 转换为列表并排序
|
||
formatted_list = list(formatted_data.values())
|
||
formatted_list.sort(key=lambda x: (x['area_name'], x['year'], x['month']))
|
||
|
||
return formatted_list
|
||
|
||
def update_predictions(original_data_path, new_data_path):
|
||
"""
|
||
使用新数据更新预测
|
||
original_data_path: 原始数据文件路径
|
||
new_data_path: 新数据文件路径
|
||
"""
|
||
try:
|
||
# 读取原始数据和新数据
|
||
original_df = pd.read_excel(original_data_path)
|
||
new_df = pd.read_excel(new_data_path)
|
||
|
||
# 确保新数据格式与原始数据一致
|
||
required_cols = set(original_df.columns)
|
||
if not set(new_df.columns).issuperset(required_cols):
|
||
raise ValueError("New data missing required columns")
|
||
|
||
# 合并数据并去重（保留最新）
|
||
updated_df = pd.concat([original_df, new_df], ignore_index=True)
|
||
updated_df = updated_df.drop_duplicates(
|
||
subset=['area_name', 'year', 'month'],
|
||
keep='last'
|
||
).reset_index(drop=True)
|
||
|
||
# 初始化预测器
|
||
predictor = RidgePredictor()
|
||
|
||
# 存储所有结果
|
||
all_results = []
|
||
all_importance = []
|
||
all_metrics = []
|
||
all_future_results = []
|
||
|
||
# 获取所有地区和项目
|
||
areas = updated_df['area_name'].unique()
|
||
projects = ['xxzs', 'zfjc', 'tsjb', 'xzcf', 'cckh', 'cjjc', 'zhxzb']
|
||
|
||
# 获取最新的时间点
|
||
latest_date = pd.to_datetime(
|
||
updated_df['year'].astype(str) + '-' +
|
||
updated_df['month'].astype(str) + '-01'
|
||
).max()
|
||
|
||
# 对每个地区进行预测
|
||
for area in areas:
|
||
print(f"Updating predictions for {area}")
|
||
current_area_importance = None
|
||
|
||
for project in projects:
|
||
print(f"Processing project: {project}")
|
||
|
||
# 使用更新后的数据重新训练并预测
|
||
results, importance, metrics = predictor.predict_with_history(
|
||
updated_df,
|
||
area,
|
||
project,
|
||
start_year=latest_date.year,
|
||
start_month=latest_date.month,
|
||
end_month=latest_date.month
|
||
)
|
||
all_results.extend(results)
|
||
|
||
# 更新特征重要性
|
||
if current_area_importance is None:
|
||
current_area_importance = importance
|
||
else:
|
||
current_area_importance = [
|
||
dict(current, **new)
|
||
for current, new in zip(current_area_importance, importance)
|
||
]
|
||
|
||
all_metrics.extend(metrics)
|
||
|
||
# 预测未来3个月
|
||
future_results = predictor.predict_future(
|
||
updated_df,
|
||
area,
|
||
project,
|
||
start_month=latest_date.month + 1,
|
||
end_month=latest_date.month + 3,
|
||
year=latest_date.year
|
||
)
|
||
all_future_results.extend(future_results)
|
||
|
||
if current_area_importance:
|
||
all_importance.extend(current_area_importance)
|
||
|
||
# 保存更新后的结果
|
||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||
|
||
# 保存更新后的完整数据集
|
||
updated_df.to_excel(f'updated_dataset_{timestamp}.xlsx', index=False)
|
||
|
||
# 保存新的预测结果
|
||
pd.DataFrame(all_results).to_excel(
|
||
f'updated_predictions_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
# 保存新的特征重要性
|
||
importance_df = pd.DataFrame(all_importance)
|
||
cols = ['area', 'year', 'month']
|
||
feature_cols = [col for col in importance_df.columns if col not in cols]
|
||
importance_df = importance_df[cols + sorted(feature_cols)]
|
||
importance_df.to_excel(
|
||
f'updated_feature_importance_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
# 保存新的模型评估指标
|
||
pd.DataFrame(all_metrics).to_excel(
|
||
f'updated_model_metrics_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
# 保存新的未来预测结果
|
||
pd.DataFrame(all_future_results).to_excel(
|
||
f'updated_future_predictions_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
print("Update and prediction completed successfully!")
|
||
return updated_df, all_results, all_future_results
|
||
|
||
except Exception as e:
|
||
print(f"Error in update_predictions: {str(e)}")
|
||
raise
|
||
|
||
def main():
|
||
try:
|
||
# 读取数据
|
||
# df = pd.read_excel('test.xlsx')
|
||
df = pd.read_csv('H:\公司-文件\公司-维域-文件\buliang-main\buliang\src\main\resources\python\data.csv')
|
||
|
||
# 初始化预测器
|
||
predictor = RidgePredictor()
|
||
|
||
# 存储所有结果
|
||
all_results = []
|
||
all_importance = []
|
||
all_metrics = []
|
||
all_future_results = []
|
||
|
||
# 获取所有地区和项目
|
||
areas = df['area_name'].unique()
|
||
projects = ['xxzs', 'zfjc', 'tsjb', 'xzcf', 'cckh', 'cjjc', 'zhxzb']
|
||
|
||
# 对每个地区进行预测
|
||
for area in areas:
|
||
print(f"Processing{area}")
|
||
# 初始化当前地区的特征重要性列表
|
||
current_area_importance = None
|
||
|
||
# 对每个项目进行预测
|
||
for project in projects:
|
||
print(f"Processing project: {project}")
|
||
|
||
# 预测2023.1-6
|
||
results, importance, metrics = predictor.predict_with_history(
|
||
df, area, project
|
||
)
|
||
all_results.extend(results)
|
||
|
||
# 更新当前地区的特征重要性
|
||
if current_area_importance is None:
|
||
current_area_importance = importance
|
||
else:
|
||
# 合并特征重要性
|
||
current_area_importance = [
|
||
dict(current, **new)
|
||
for current, new in zip(current_area_importance, importance)
|
||
]
|
||
|
||
all_metrics.extend(metrics)
|
||
|
||
# 预测2023.7-9
|
||
future_results = predictor.predict_future(df, area, project)
|
||
all_future_results.extend(future_results)
|
||
|
||
# 将该地区的所有特征重要性添加到总结果中
|
||
if current_area_importance:
|
||
all_importance.extend(current_area_importance)
|
||
|
||
# 保存结果
|
||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||
# 转换预测结果格式
|
||
def format_predictions(results_list):
|
||
formatted_data = {}
|
||
for result in results_list:
|
||
key = (result['area'], result['year'], result['month'])
|
||
if key not in formatted_data:
|
||
formatted_data[key] = {
|
||
'area_name': result['area'],
|
||
'year': result['year'],
|
||
'month': result['month'],
|
||
'xxzs': 0,
|
||
'zfjc': 0,
|
||
'tsjb': 0,
|
||
'xzcf': 0,
|
||
'cckh': 0,
|
||
'cjjc': 0,
|
||
'zhxzb': 0
|
||
}
|
||
formatted_data[key][result['project']] = round(result['prediction'] , 8)
|
||
return list(formatted_data.values())
|
||
|
||
# 合并并保存所有预测结果
|
||
all_predictions = format_predictions(all_results + all_future_results)
|
||
predictions_df = pd.DataFrame(all_predictions)
|
||
predictions_df = predictions_df[['area_name', 'year', 'month',
|
||
'xxzs', 'zfjc', 'tsjb', 'xzcf',
|
||
'cckh', 'cjjc', 'zhxzb']]
|
||
predictions_df.to_excel(f'all_predictions_{timestamp}.xlsx', index=False)
|
||
|
||
|
||
# 保存1-6月预测结果
|
||
pd.DataFrame(all_results).to_excel(
|
||
f'predictions_1_6_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
# 保存特征重要性
|
||
importance_df = pd.DataFrame(all_importance)
|
||
# 确保列的顺序
|
||
cols = ['area', 'year', 'month']
|
||
feature_cols = [col for col in importance_df.columns if col not in cols]
|
||
importance_df = importance_df[cols + sorted(feature_cols)]
|
||
importance_df.to_excel(
|
||
f'feature_importance_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
# 保存模型评估指标
|
||
pd.DataFrame(all_metrics).to_excel(
|
||
f'model_metrics_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
# 保存7-9月预测结果
|
||
pd.DataFrame(all_future_results).to_excel(
|
||
f'predictions_7_9_{timestamp}.xlsx', index=False
|
||
)
|
||
|
||
print("Prediction completed successfully!")
|
||
|
||
# 当有新数据时，可以运行更新预测
|
||
# update_predictions('test.xlsx', 'new_data.xlsx')
|
||
except Exception as e:
|
||
print(f"Error in main function: {str(e)}")
|
||
raise
|
||
|
||
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |