508 lines
21 KiB
Python
508 lines
21 KiB
Python
import pandas as pd
|
||
import numpy as np
|
||
from sklearn.preprocessing import StandardScaler
|
||
from sklearn.preprocessing import PolynomialFeatures
|
||
from sklearn.linear_model import Ridge
|
||
from sklearn.model_selection import GridSearchCV, TimeSeriesSplit
|
||
from sklearn.metrics import r2_score
|
||
import plotly.graph_objects as go
|
||
from plotly.subplots import make_subplots
|
||
import os
|
||
import joblib
|
||
import json
|
||
import sys
|
||
|
||
|
||
# 重新配置标准输出流的编码为 UTF-8
|
||
sys.stdout.reconfigure(encoding='utf-8')
|
||
|
||
class ComprehensiveScoreAnalyzer:
|
||
def __init__(self):
|
||
self.project_models = {}
|
||
self.final_model = None
|
||
self.feature_scalers = {}
|
||
self.score_scaler = StandardScaler()
|
||
self.project_polys = {}
|
||
self.score_poly = None
|
||
self.project_names = ['xxzs', 'zfjc', 'tsjb', 'xzcf', 'cckh', 'cjjc', 'zhxzb']
|
||
self.feature_importance = {}
|
||
|
||
# 定义每个项目的特征前缀
|
||
self.project_features = {
|
||
'xxzs': 'xxzs_',
|
||
'zfjc': 'zfjc_',
|
||
'tsjb': 'tsjb_',
|
||
'xzcf': 'xzcf_',
|
||
'cckh': 'cckh_',
|
||
'cjjc': 'cjjc_',
|
||
'zhxzb': 'zhxzb_'
|
||
}
|
||
|
||
#
|
||
# def preprocess_data(self, df):
|
||
# df['date'] = pd.to_datetime(df['year'].astype(str) + '-' + df['month'].astype(str))
|
||
# features = {}
|
||
# for project, prefix in self.project_features.items():
|
||
# project_cols = [col for col in df.columns if col.startswith(prefix) and col != project]
|
||
# if project_cols:
|
||
# features[project] = df[project_cols]
|
||
# valid_projects = []
|
||
# project_scores = pd.DataFrame()
|
||
# for project in self.project_names:
|
||
# if df[project].std() > 0 and project in features and not features[project].empty:
|
||
# valid_projects.append(project)
|
||
# project_scores[project] = df[project]
|
||
# else:
|
||
# # print(f"警告: {project} 项目得分无变化或无子特征，已剔除")
|
||
# self.project_names = valid_projects
|
||
# total_score = df['all_score']
|
||
# return features, project_scores, total_score, df['date']
|
||
def preprocess_data(self, df):
|
||
df['date'] = pd.to_datetime(df['year'].astype(str) + '-' + df['month'].astype(str))
|
||
features = {}
|
||
for project, prefix in self.project_features.items():
|
||
project_cols = [col for col in df.columns if col.startswith(prefix) and col != project]
|
||
if project_cols:
|
||
features[project] = df[project_cols]
|
||
project_scores = pd.DataFrame()
|
||
for project in self.project_names:
|
||
if project in features and not features[project].empty:
|
||
project_scores[project] = df[project]
|
||
# else:
|
||
# print(f"警告: {project} 项目无子特征或没有数据")
|
||
noise = np.random.normal(0, 0.01, project_scores.shape)
|
||
project_scores += noise
|
||
total_score = df['all_score']
|
||
|
||
return features, project_scores, total_score, df['date']
|
||
|
||
def optimize_model(self, X, y, project_name=None):
|
||
param_grid = {
|
||
'alpha': [0.001, 0.01, 0.1, 1.0, 10.0, 100.0],
|
||
'fit_intercept': [True],
|
||
'solver': ['auto']
|
||
}
|
||
tscv = TimeSeriesSplit(n_splits=5)
|
||
grid_search = GridSearchCV(
|
||
Ridge(),
|
||
param_grid,
|
||
cv=tscv,
|
||
scoring='neg_mean_squared_error',
|
||
n_jobs=1,
|
||
verbose=0
|
||
)
|
||
grid_search.fit(X, y)
|
||
# print(f"{'最终' if project_name is None else project_name} 模型最佳参数:", grid_search.best_params_)
|
||
return grid_search.best_estimator_
|
||
|
||
def fit_project_models(self, features, project_scores, train_mask):
|
||
for project in self.project_names:
|
||
# print(f"\n训练 {project} 项目模型:")
|
||
project_features = features[project]
|
||
self.feature_scalers[project] = StandardScaler()
|
||
self.project_polys[project] = PolynomialFeatures(degree=2, include_bias=False)
|
||
X_train = self.feature_scalers[project].fit_transform(project_features[train_mask])
|
||
X_train_poly = self.project_polys[project].fit_transform(X_train)
|
||
y_train = project_scores[project][train_mask]
|
||
model = self.optimize_model(X_train_poly, y_train, project)
|
||
self.project_models[project] = model
|
||
importance = pd.DataFrame({
|
||
'feature': self.project_polys[project].get_feature_names_out(project_features.columns),
|
||
'coefficient': abs(model.coef_)
|
||
}).sort_values('coefficient', ascending=False)
|
||
self.feature_importance[project] = importance
|
||
|
||
def fit_final_model(self, project_scores, total_score, train_mask):
|
||
# print("\n训练最终总分模型:")
|
||
X_scores = self.score_scaler.fit_transform(project_scores[train_mask])
|
||
self.score_poly = PolynomialFeatures(degree=2, include_bias=False)
|
||
X_scores_poly = self.score_poly.fit_transform(X_scores)
|
||
self.final_model = self.optimize_model(X_scores_poly, total_score[train_mask])
|
||
|
||
def predict(self, features, project_scores, test_mask):
|
||
project_predictions = pd.DataFrame()
|
||
for project in self.project_names:
|
||
project_features = features[project]
|
||
X_test = self.feature_scalers[project].transform(project_features[test_mask])
|
||
X_test_poly = self.project_polys[project].transform(X_test)
|
||
project_predictions[project] = self.project_models[project].predict(X_test_poly)
|
||
X_scores_test = self.score_scaler.transform(project_scores[test_mask])
|
||
X_scores_test_poly = self.score_poly.transform(X_scores_test)
|
||
total_score_pred = self.final_model.predict(X_scores_test_poly)
|
||
return project_predictions, total_score_pred
|
||
|
||
def plot_project_comparisons(self, features, project_scores, train_mask):
|
||
n_projects = len(self.project_names)
|
||
n_cols = 3
|
||
n_rows = (n_projects + n_cols - 1) // n_cols
|
||
fig = make_subplots(
|
||
rows=n_rows,
|
||
cols=n_cols,
|
||
subplot_titles=self.project_names,
|
||
vertical_spacing=0.22,
|
||
horizontal_spacing=0.1
|
||
)
|
||
for i, project in enumerate(self.project_names):
|
||
row = i // n_cols + 1
|
||
col = i % n_cols + 1
|
||
project_features = features[project]
|
||
y_true = project_scores[project][train_mask]
|
||
y_pred = self.project_models[project].predict(
|
||
self.project_polys[project].transform(
|
||
self.feature_scalers[project].transform(project_features[train_mask])
|
||
)
|
||
)
|
||
r2 = r2_score(y_true, y_pred)
|
||
fig.add_trace(
|
||
go.Scatter(
|
||
x=y_true,
|
||
y=y_pred,
|
||
mode='markers',
|
||
name=project,
|
||
marker=dict(
|
||
size=8,
|
||
color='rgba(8, 81, 156, 0.6)',
|
||
line=dict(
|
||
color='rgba(8, 81, 156, 0.9)',
|
||
width=1
|
||
)
|
||
),
|
||
showlegend=False
|
||
),
|
||
row=row, col=col
|
||
)
|
||
|
||
min_val = min(min(y_true), min(y_pred))
|
||
max_val = max(max(y_true), max(y_pred))
|
||
fig.add_trace(
|
||
go.Scatter(
|
||
x=[min_val, max_val],
|
||
y=[min_val, max_val],
|
||
mode='lines',
|
||
line=dict(color='red', dash='dash'),
|
||
showlegend=False
|
||
),
|
||
row=row, col=col
|
||
)
|
||
fig.add_annotation(
|
||
text=f'R² = {r2:.4f}',
|
||
xref=f'x{i + 1}',
|
||
yref=f'y{i + 1}',
|
||
x=0.05,
|
||
y=0.95,
|
||
showarrow=False,
|
||
font=dict(size=10),
|
||
xanchor='left',
|
||
yanchor='top'
|
||
)
|
||
pad = 0.1 * (max_val - min_val)
|
||
fig.update_xaxes(range=[min_val - pad, max_val + pad], title_text="实际值", row=row, col=col)
|
||
fig.update_yaxes(range=[min_val - pad, max_val + pad], title_text="预测值", row=row, col=col)
|
||
fig.update_layout(
|
||
height=300 * n_rows,
|
||
width=1000,
|
||
title_text="各项目得分预测结果对比",
|
||
showlegend=False,
|
||
template='plotly_white',
|
||
)
|
||
return fig
|
||
|
||
def plot_total_score_comparison(self, project_scores, total_score, train_mask):
|
||
y_true = total_score[train_mask]
|
||
y_pred = self.final_model.predict(
|
||
self.score_poly.transform(
|
||
self.score_scaler.transform(project_scores[train_mask])))
|
||
r2 = r2_score(y_true, y_pred)
|
||
fig = go.Figure()
|
||
fig.add_trace(
|
||
go.Scatter(
|
||
x=y_true,
|
||
y=y_pred,
|
||
mode='markers',
|
||
name='预测值',
|
||
marker=dict(
|
||
size=10,
|
||
color='rgba(255, 0, 0, 0.6)',
|
||
line=dict(
|
||
color='rgba(255, 0, 0, 0.9)',
|
||
width=1
|
||
)
|
||
)
|
||
)
|
||
)
|
||
min_val = min(min(y_true), min(y_pred))
|
||
max_val = max(max(y_true), max(y_pred))
|
||
fig.add_trace(
|
||
go.Scatter(
|
||
x=[min_val, max_val],
|
||
y=[min_val, max_val],
|
||
mode='lines',
|
||
line=dict(color='red', dash='dash'),
|
||
name='y=x'
|
||
)
|
||
)
|
||
fig.add_annotation(
|
||
text=f'R² = {r2:.4f}',
|
||
xref='paper',
|
||
yref='paper',
|
||
x=0.05,
|
||
y=0.95,
|
||
showarrow=False,
|
||
font=dict(size=14)
|
||
)
|
||
fig.update_layout(
|
||
title_text="总分预测结果对比",
|
||
xaxis_title="实际值",
|
||
yaxis_title="预测值",
|
||
template='plotly_white',
|
||
width=800,
|
||
height=600
|
||
)
|
||
return fig
|
||
|
||
def save_models(self, area_name, output_dir='models'):
|
||
if not os.path.exists(output_dir):
|
||
os.makedirs(output_dir)
|
||
area_dir = os.path.join(output_dir, area_name)
|
||
if not os.path.exists(area_dir):
|
||
os.makedirs(area_dir)
|
||
for project in self.project_names:
|
||
model_data = {
|
||
'model': self.project_models[project],
|
||
'scaler': self.feature_scalers[project],
|
||
'poly': self.project_polys[project]
|
||
}
|
||
model_path = os.path.join(area_dir, f'{project}_model.pkl')
|
||
joblib.dump(model_data, model_path)
|
||
final_model_data = {
|
||
'model': self.final_model,
|
||
'scaler': self.score_scaler,
|
||
'poly': self.score_poly
|
||
}
|
||
final_model_path = os.path.join(area_dir, 'final_model.pkl')
|
||
joblib.dump(final_model_data, final_model_path)
|
||
|
||
|
||
def process_feature_importance(analyzer, area):
|
||
importance_dict = {'area_name': area}
|
||
for project in analyzer.project_names:
|
||
importance_df = analyzer.feature_importance[project]
|
||
original_features = importance_df[
|
||
~importance_df['feature'].str.contains(' ')
|
||
].copy()
|
||
for _, row in original_features.iterrows():
|
||
importance_dict[row['feature']] = row['coefficient']
|
||
|
||
return importance_dict
|
||
|
||
|
||
def run_prediction(train_date, test_date, period_months):
|
||
period_dir = f"{period_months}个月"
|
||
for base_dir in ['models', 'predictions', 'feature_importance', 'plots']:
|
||
period_path = os.path.join(base_dir, period_dir)
|
||
if not os.path.exists(period_path):
|
||
os.makedirs(period_path)
|
||
try:
|
||
predictions, feature_importance = main(train_date, test_date, period_dir)
|
||
# print(f"\n{period_months}个月预测完成！")
|
||
|
||
# 创建一个包含 predictions 和 feature_importance 的字典
|
||
result = {
|
||
"predictions": json.loads(predictions.to_json(orient='records')),
|
||
"feature_importance": json.loads(feature_importance.to_json(orient='records'))
|
||
}
|
||
|
||
# 使用 json.dumps 将字典转换为 JSON 格式的字符串
|
||
result_json = json.dumps(result, ensure_ascii=False, indent=4)
|
||
print(result_json)
|
||
# if predictions is not None:
|
||
|
||
# print(f"共生成 {len(predictions)} 条预测记录")
|
||
# if feature_importance is not None:
|
||
# print(f"特征重要性分析包含 {len(feature_importance)} 条记录")
|
||
except Exception as e:
|
||
print(f"{period_months}个月预测执行出错: {str(e)}")
|
||
def main(train_date='2023-01-01', test_date='2023-06-01', period_dir='6个月'):
|
||
"""
|
||
主函数，用于执行数据分析和预测流程。
|
||
|
||
参数:
|
||
- train_date (str): 训练数据的截止日期，默认为'2023-01-01'。
|
||
- test_date (str): 测试数据的起始日期，默认为'2023-06-01'。
|
||
- period_dir (str): 保存模型、预测结果和特征重要性等的目录名称，默认为'6个月'。
|
||
|
||
返回:
|
||
- all_predictions_df (pd.DataFrame): 所有地区的预测结果数据框，如果未生成则为None。
|
||
- importance_df (pd.DataFrame): 所有地区的特征重要性数据框，如果未生成则为None。
|
||
"""
|
||
# 读取数据集
|
||
# df = pd.read_csv('data.csv')
|
||
df = pd.read_csv('H:/develop/dama/java/buliangfanying/target/classes/python/data.csv')
|
||
# 获取所有不同的地区名称
|
||
areas = df['area_name'].unique()
|
||
# 初始化存储所有预测结果和特征重要性的列表
|
||
all_predictions = []
|
||
all_feature_importance = []
|
||
# 识别特征列，即以特定前缀开头的列
|
||
feature_columns = [col for col in df.columns if any(col.startswith(prefix)
|
||
for prefix in ['xxzs_', 'zfjc_', 'tsjb_', 'xzcf_',
|
||
'cckh_', 'cjjc_', 'zhxzb_'])]
|
||
# 遍历每个地区
|
||
for area in areas:
|
||
# 打印地区分割线
|
||
# print(f"\n{'=' * 50}")
|
||
# print(f"处理地区: {area}")
|
||
# print(f"{'=' * 50}")
|
||
# 筛选特定地区的数据
|
||
area_df = df[df['area_name'] == area].copy()
|
||
# 初始化 ComprehensiveScoreAnalyzer 实例
|
||
analyzer = ComprehensiveScoreAnalyzer()
|
||
# 预处理数据
|
||
features, project_scores, total_score, dates = analyzer.preprocess_data(area_df)
|
||
# 创建训练集和测试集的掩码
|
||
train_mask = dates < pd.to_datetime(train_date)
|
||
test_mask = dates >= pd.to_datetime(test_date)
|
||
# 检查训练集和测试集的数据量是否充足
|
||
if sum(train_mask) < 2 or sum(test_mask) < 1:
|
||
# print(f"警告: {area} 数据量不足，跳过此地区")
|
||
continue
|
||
try:
|
||
# 训练项目模型
|
||
analyzer.fit_project_models(features, project_scores, train_mask)
|
||
# 训练最终模型
|
||
analyzer.fit_final_model(project_scores, total_score, train_mask)
|
||
# 保存模型
|
||
model_dir = os.path.join('models', period_dir)
|
||
analyzer.save_models(area, output_dir=model_dir)
|
||
# 处理并存储特征重要性
|
||
importance_dict = process_feature_importance(analyzer, area)
|
||
all_feature_importance.append(importance_dict)
|
||
# 进行预测
|
||
project_pred, total_pred = analyzer.predict(features, project_scores, test_mask)
|
||
# 准备测试数据以保存预测结果
|
||
test_data = area_df[test_mask].copy()
|
||
for project in analyzer.project_names:
|
||
test_data[project] = project_pred[project].values
|
||
test_data['all_score'] = total_pred
|
||
all_predictions.append(test_data)
|
||
# 创建并保存可视化图表
|
||
plots_dir = os.path.join('plots', period_dir)
|
||
if not os.path.exists(plots_dir):
|
||
os.makedirs(plots_dir)
|
||
project_fig = analyzer.plot_project_comparisons(features, project_scores, train_mask)
|
||
project_fig.write_image(os.path.join(plots_dir, f"{area}_project_comparisons.png"))
|
||
total_fig = analyzer.plot_total_score_comparison(project_scores, total_score, train_mask)
|
||
total_fig.write_image(os.path.join(plots_dir, f"{area}_total_score_comparison.png"))
|
||
except Exception as e:
|
||
# 错误处理
|
||
# print(f"处理 {area} 时发生错误: {str(e)}")
|
||
continue
|
||
# 合并所有地区的预测结果
|
||
if all_predictions:
|
||
all_predictions_df = pd.concat(all_predictions, ignore_index=True)
|
||
all_predictions_df = all_predictions_df.sort_values(['area_name', 'year', 'month'])
|
||
predictions_dir = os.path.join('predictions', period_dir)
|
||
all_predictions_df.to_csv(os.path.join(predictions_dir, 'all_areas_predictions.csv'), index=False)
|
||
# 合并所有地区的特征重要性
|
||
if all_feature_importance:
|
||
importance_df = pd.DataFrame(all_feature_importance)
|
||
for col in feature_columns:
|
||
if col not in importance_df.columns:
|
||
importance_df[col] = 0.0
|
||
cols_order = ['area_name'] + [col for col in feature_columns if col in importance_df.columns]
|
||
importance_df = importance_df[cols_order]
|
||
importance_dir = os.path.join('feature_importance', period_dir)
|
||
importance_df.to_csv(os.path.join(importance_dir, 'all_areas_feature_importance.csv'), index=False)
|
||
# 打印保存路径信息
|
||
# print(f"\n所有结果已保存在 {period_dir} 目录下：")
|
||
# if all_predictions:
|
||
# print(f"1. 预测结果：predictions/{period_dir}/all_areas_predictions.csv")
|
||
# if all_feature_importance:
|
||
# print(f"2. 特征重要性：feature_importance/{period_dir}/all_areas_feature_importance.csv")
|
||
# print(f"3. 模型保存在 models/{period_dir} 目录下")
|
||
# print(f"4. 可视化结果保存在 plots/{period_dir} 目录下")
|
||
# 返回预测结果和特征重要性的数据框
|
||
return all_predictions_df if all_predictions else None, \
|
||
importance_df if all_feature_importance else None
|
||
|
||
|
||
def load_model(area_name, project=None, base_dir='models'):
|
||
area_dir = os.path.join(base_dir, area_name)
|
||
if not os.path.exists(area_dir):
|
||
raise FileNotFoundError(f"未找到地区 {area_name} 的模型目录")
|
||
if project is None:
|
||
model_path = os.path.join(area_dir, 'final_model.pkl')
|
||
else:
|
||
model_path = os.path.join(area_dir, f'{project}_model.pkl')
|
||
if not os.path.exists(model_path):
|
||
raise FileNotFoundError(f"未找到模型文件: {model_path}")
|
||
return joblib.load(model_path)
|
||
|
||
|
||
def predict_scores(area_name, features_df, base_dir='models'):
|
||
project_predictions = {}
|
||
project_scores = pd.DataFrame()
|
||
area_dir = os.path.join(base_dir, area_name)
|
||
model_files = [f for f in os.listdir(area_dir) if f.endswith('_model.pkl')]
|
||
for model_file in model_files:
|
||
if model_file == 'final_model.pkl':
|
||
continue
|
||
project = model_file.replace('_model.pkl', '')
|
||
model_data = load_model(area_name, project)
|
||
project_prefix = project + '_'
|
||
project_features = features_df[[col for col in features_df.columns if col.startswith(project_prefix)]]
|
||
X = model_data['scaler'].transform(project_features)
|
||
X_poly = model_data['poly'].transform(X)
|
||
project_predictions[project] = model_data['model'].predict(X_poly)
|
||
project_scores[project] = project_predictions[project]
|
||
final_model_data = load_model(area_name)
|
||
X_scores = final_model_data['scaler'].transform(project_scores)
|
||
X_scores_poly = final_model_data['poly'].transform(X_scores)
|
||
total_score_prediction = final_model_data['model'].predict(X_scores_poly)
|
||
return project_predictions, total_score_prediction
|
||
|
||
|
||
if __name__ == "__main__":
|
||
|
||
# file_path = sys.argv[1]
|
||
file_path = 'H:/develop/dama/java/buliangfanying/PredictedScores.txt'
|
||
with open(file_path, 'r', encoding='utf-8') as file:
|
||
json_str = file.read()
|
||
|
||
# 解析 JSON 字符串
|
||
data_dict = json.loads(json_str)
|
||
|
||
base_dirs = ['models', 'predictions', 'feature_importance', 'plots']
|
||
for dir_name in base_dirs:
|
||
if not os.path.exists(dir_name):
|
||
os.makedirs(dir_name)
|
||
prediction_configs = [
|
||
|
||
{
|
||
'train_date': data_dict['train_date'], # 使用6月份以前的数据
|
||
'test_date': data_dict['test_date'], # 预测6月份
|
||
'period_months': data_dict['period_months']
|
||
}
|
||
# {
|
||
# 'train_date': '2023-06-01', # 使用6月份以前的数据
|
||
# 'test_date': '2023-06-01', # 预测6月份
|
||
# 'period_months': 1
|
||
# }
|
||
# {
|
||
# 'train_date': '2023-04-01', # 使用4月份以前的数据
|
||
# 'test_date': '2023-06-01', # 预测6月份
|
||
# 'period_months': 3
|
||
# },
|
||
# {
|
||
# 'train_date': '2023-01-01', # 使用1月份以前的数据
|
||
# 'test_date': '2023-06-01', # 预测6月份
|
||
# 'period_months': 6
|
||
# }
|
||
]
|
||
for config in prediction_configs:
|
||
run_prediction(
|
||
config['train_date'],
|
||
config['test_date'],
|
||
config['period_months']
|
||
)
|