全局修正
This commit is contained in:
@ -19,6 +19,7 @@ from sklearn.cross_decomposition import PLSRegression
|
||||
from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor
|
||||
from sklearn.tree import DecisionTreeRegressor
|
||||
from sklearn.neural_network import MLPRegressor
|
||||
from sklearn.pipeline import Pipeline
|
||||
from joblib import parallel_backend
|
||||
# 第三方模型导入
|
||||
# try:
|
||||
@ -44,7 +45,7 @@ import os
|
||||
is_frozen_env = getattr(sys, 'frozen', False)
|
||||
safe_n_jobs = 1 if is_frozen_env else -1
|
||||
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
|
||||
from src.core.utils.split_methods import spxy, ks
|
||||
|
||||
|
||||
@ -454,25 +455,28 @@ class WaterQualityModelingBatch:
|
||||
else:
|
||||
raise ValueError(f"不支持的划分方法: {method}. 支持的方法: {self.split_methods}")
|
||||
|
||||
def train_single_model(self, X: np.ndarray, y: pd.Series, model_name: str,
|
||||
def train_single_model(self, X_raw: pd.DataFrame, y: pd.Series, model_name: str,
|
||||
cv_folds: int = 5, scoring: str = 'neg_mean_squared_error',
|
||||
test_size: float = 0.2, random_state: int = 42,
|
||||
split_method: str = "random") -> Dict:
|
||||
split_method: str = "random",
|
||||
preprocess_method: str = "None") -> Dict:
|
||||
"""
|
||||
训练单个回归模型
|
||||
训练单个回归模型（Pipeline 化：preprocess_method 字符串内部构造 Pipeline，
|
||||
scaler/MSC.mean_spectrum_ 等状态被绑定在 best_model 上，CV 与 test 评估
|
||||
都在「只拟合训练 fold 的 scaler」之上，避免传统「X_full → scaler.fit →
|
||||
split → CV」造成的数据泄露）。
|
||||
|
||||
Args:
|
||||
X: 特征数据
|
||||
X_raw: 原始特征数据（未经预处理，DataFrame 形态方便 Pipeline 内部转换）
|
||||
y: 目标值数据
|
||||
model_name: 模型名称
|
||||
cv_folds: 交叉验证折数
|
||||
scoring: 评分指标
|
||||
test_size: 测试集比例
|
||||
random_state: 随机种子
|
||||
split_method: 数据划分方法
|
||||
preprocess_method: 预处理方法字符串（如 'None' / 'SS' / 'MSC'）；
|
||||
训练结束后 best_model 字段即为 sklearn Pipeline。
|
||||
其余参数（cv_folds / scoring / test_size / random_state / split_method）
|
||||
含义保持不变。
|
||||
|
||||
Returns:
|
||||
训练结果字典
|
||||
训练结果字典；'model' 字段现在是 sklearn.pipeline.Pipeline（含 scaler）
|
||||
"""
|
||||
if model_name not in self.model_configs:
|
||||
raise ValueError(f"不支持的模型: {model_name}")
|
||||
@ -483,18 +487,18 @@ class WaterQualityModelingBatch:
|
||||
print(f"模型 {model_name} 不可用，请安装相应的库")
|
||||
return None
|
||||
|
||||
print(f"开始训练模型: {model_name}")
|
||||
print(f"开始训练模型: {model_name} (预处理: {preprocess_method})")
|
||||
|
||||
# 使用指定方法分割训练集和测试集
|
||||
# 使用指定方法分割训练集和测试集（用原始 X_raw，Pipeline 内置 transform 处理）
|
||||
X_train, X_test, y_train, y_test = self.split_data(
|
||||
X, y, method=split_method, test_size=test_size, random_state=random_state
|
||||
X_raw, y, method=split_method, test_size=test_size, random_state=random_state
|
||||
)
|
||||
|
||||
print(f"数据分割完成:")
|
||||
print(f" 训练集样本数: {X_train.shape[0]}")
|
||||
print(f" 测试集样本数: {X_test.shape[0]}")
|
||||
|
||||
# 创建模型实例
|
||||
# 构造 base_model
|
||||
if callable(config['model']):
|
||||
base_model = config['model']()
|
||||
else:
|
||||
@ -506,12 +510,25 @@ class WaterQualityModelingBatch:
|
||||
elif model_name == 'LightGBM':
|
||||
base_model.set_params(verbose=-1)
|
||||
|
||||
# 随机搜索 —— 替代穷举式 GridSearchCV，大幅降低寻优时间
|
||||
# ============ 关键：把预处理器塞进 Pipeline ============
|
||||
preproc = get_preprocessing_transformer(preprocess_method)
|
||||
pipeline = Pipeline([
|
||||
('preproc', preproc),
|
||||
('model', base_model),
|
||||
])
|
||||
|
||||
# RandomizedSearchCV 需要以「步骤名__参数名」的格式索引参数网格；
|
||||
# 我们原有的 config['params'] 是模型层的（无 __），统一加 model__ 前缀。
|
||||
prefixed_params = {
|
||||
f"model__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
|
||||
# 随机搜索：直接对 Pipeline 调优（scaler 仅在 train fold 上 fit）
|
||||
cv_strategy = KFold(n_splits=cv_folds, shuffle=True, random_state=random_state)
|
||||
|
||||
grid_search = RandomizedSearchCV(
|
||||
base_model,
|
||||
config['params'],
|
||||
pipeline,
|
||||
prefixed_params,
|
||||
n_iter=10,
|
||||
cv=cv_strategy,
|
||||
scoring=scoring,
|
||||
@ -522,20 +539,22 @@ class WaterQualityModelingBatch:
|
||||
|
||||
grid_search.fit(X_train, y_train)
|
||||
|
||||
# 获取最佳模型
|
||||
# 获取最佳模型（已是 Pipeline）
|
||||
best_model = grid_search.best_estimator_
|
||||
|
||||
# 交叉验证评估（在训练集上）
|
||||
cv_scores = cross_val_score(best_model, X_train, y_train, cv=cv_strategy, scoring=scoring)
|
||||
# 交叉验证评估（在训练集上）：cross_val_score 会对 Pipeline 重 clone，
|
||||
# 保证每个 fold 重 fit 预处理，CV 评分反映「无泄露」真实泛化能力
|
||||
cv_scores = cross_val_score(best_model, X_train, y_train, cv=cv_strategy,
|
||||
scoring=scoring, n_jobs=safe_n_jobs)
|
||||
|
||||
# 计算训练集上的回归指标
|
||||
# 计算训练集上的回归指标（Pipeline 内 fit_transform 只发生一次，已 fit 完毕）
|
||||
y_train_pred = best_model.predict(X_train)
|
||||
train_mse = mean_squared_error(y_train, y_train_pred)
|
||||
train_mae = mean_absolute_error(y_train, y_train_pred)
|
||||
train_r2 = r2_score(y_train, y_train_pred)
|
||||
train_rmse = np.sqrt(train_mse)
|
||||
|
||||
# 计算测试集上的回归指标
|
||||
# 计算测试集上的回归指标（用训练集 fit 出的 scaler，正确的 deploy-time 行为）
|
||||
y_test_pred = best_model.predict(X_test)
|
||||
test_mse = mean_squared_error(y_test, y_test_pred)
|
||||
test_mae = mean_absolute_error(y_test, y_test_pred)
|
||||
@ -562,7 +581,10 @@ class WaterQualityModelingBatch:
|
||||
# 数据分割信息
|
||||
'train_size': X_train.shape[0],
|
||||
'test_size': X_test.shape[0],
|
||||
'split_method': split_method
|
||||
'split_method': split_method,
|
||||
# Pipeline 信息（用于诊断 / metadata）
|
||||
'preprocess_method': preprocess_method,
|
||||
'is_pipeline': isinstance(best_model, Pipeline),
|
||||
}
|
||||
|
||||
print(f"模型 {model_name} 训练完成:")
|
||||
@ -714,21 +736,21 @@ class WaterQualityModelingBatch:
|
||||
print(f"{'-' * 60}")
|
||||
|
||||
try:
|
||||
# 数据预处理
|
||||
X_processed = self.preprocess_data(X_raw, preprocess_method)
|
||||
|
||||
# 训练模型
|
||||
result = self.train_single_model(X_processed, y, model_name,
|
||||
cv_folds, scoring, test_size, random_state, split_method)
|
||||
# 不再外部 Preprocessing——改传给 train_single_model 由 Pipeline 处理
|
||||
result = self.train_single_model(
|
||||
X_raw, y, model_name,
|
||||
cv_folds, scoring, test_size, random_state, split_method,
|
||||
preprocess_method=preprocess_method,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
# 保存模型
|
||||
# 保存模型（result['model'] 已是 sklearn Pipeline）
|
||||
metadata = {
|
||||
'target_column_name': target_column_name,
|
||||
'cv_mean': result['cv_mean'],
|
||||
'cv_std': result['cv_std'],
|
||||
'best_params': result['best_params'],
|
||||
'data_shape': X_processed.shape,
|
||||
'data_shape': X_raw.shape,
|
||||
'target_range': [float(y.min()), float(y.max())],
|
||||
'train_r2': result['train_r2'],
|
||||
'train_rmse': result['train_rmse'],
|
||||
@ -738,10 +760,13 @@ class WaterQualityModelingBatch:
|
||||
'test_mae': result['test_mae'],
|
||||
'train_size': result['train_size'],
|
||||
'test_size': result['test_size'],
|
||||
'split_method': result['split_method']
|
||||
'split_method': result['split_method'],
|
||||
# Pipeline 标记（便于旧 inference 路径兼容/诊断）
|
||||
'preprocess_method': preprocess_method,
|
||||
'is_pipeline': result.get('is_pipeline', False),
|
||||
}
|
||||
|
||||
self.save_model(result['model'], target_column_name,
|
||||
self.save_model(result['model'], target_column_name,
|
||||
f"{split_method}_{preprocess_method}",
|
||||
model_name, metadata)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user