fix: SVR+MNF 训练管线三个致命 Bug 修复
1. SVR y 尺度修复 (modeling_batch.py): - TransformedTargetRegressor 包裹 SVR,自动标准化 y 后训练 - 超参前缀 model__ → model__regressor__ (穿透包裹层) - _find_svr_in_pipeline 兼容新的 TransformedTargetRegressor 嵌套 2. MNF 噪声估计修复 (spectral_Preprocessing.py): - 旧: noise = X[1:] - X[:-1] (假设空间连续,对打乱数据错误) - 新: noise = X - savgol_filter(X) (SG 滤波残差提取纯光谱噪声) - 附带波段数检查和异常兜底 3. DualStream_MNF 补标准 (spectral_Preprocessing.py): - FeatureUnion 外再包 Pipeline + StandardScaler - MNF 主成分(±10) 和 Physical 指数(0.001) 统一量纲后再送 SVR
This commit is contained in:
@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
|
||||
from sklearn.neural_network import MLPRegressor
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.compose import TransformedTargetRegressor
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from joblib import parallel_backend
|
||||
# 第三方模型导入
|
||||
# try:
|
||||
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
|
||||
elif model_name == 'LightGBM':
|
||||
base_model.set_params(verbose=-1)
|
||||
|
||||
# ★ 致命修复 1:SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000),
|
||||
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
|
||||
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y),
|
||||
# predict 时自动 inverse_transform,对调用方完全透明。
|
||||
_is_svr = (config['model'] == SVR)
|
||||
if _is_svr:
|
||||
base_model = TransformedTargetRegressor(
|
||||
regressor=base_model,
|
||||
transformer=StandardScaler()
|
||||
)
|
||||
|
||||
# ============ 关键:把预处理器塞进 Pipeline ============
|
||||
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
|
||||
_wl_list = None
|
||||
@ -677,9 +690,16 @@ class WaterQualityModelingBatch:
|
||||
|
||||
# 以「步骤名__参数名」的格式索引参数网格;
|
||||
# config['params'] 是模型层的(无 __),统一加 model__ 前缀。
|
||||
prefixed_params = {
|
||||
f"model__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
# ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
|
||||
# GridSearchCV 参数路径变为 model__regressor__{param}
|
||||
if _is_svr:
|
||||
prefixed_params = {
|
||||
f"model__regressor__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
else:
|
||||
prefixed_params = {
|
||||
f"model__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
|
||||
# 全量网格搜索:SVR 超参组合仅 216 种(4×6×3×3),
|
||||
# 穷举远优于 RandomizedSearchCV(n_iter=10) 的随机抽样
|
||||
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
|
||||
@staticmethod
|
||||
def _find_svr_in_pipeline(pipeline):
|
||||
model_step = pipeline.named_steps.get('model')
|
||||
if model_step is not None and model_step.__class__.__name__ == 'SVR':
|
||||
if model_step is None:
|
||||
return None
|
||||
# 直接就是 SVR(旧模型 / 非 SVR 的其他回归器)
|
||||
if model_step.__class__.__name__ == 'SVR':
|
||||
return model_step
|
||||
# ★ TransformedTargetRegressor 包裹的 SVR(修复 1 引入)
|
||||
if hasattr(model_step, 'regressor_'):
|
||||
inner = model_step.regressor_
|
||||
if inner.__class__.__name__ == 'SVR':
|
||||
return inner
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
n_samples, n_features = X.shape
|
||||
|
||||
# 1) 噪声矩阵:相邻行差分
|
||||
noise = X[1:] - X[:-1]
|
||||
# 1) ★ 致命修复 3:噪声矩阵估计。
|
||||
# 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
|
||||
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
|
||||
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
|
||||
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
|
||||
# 白化操作会把真实水质信号当噪声除掉。
|
||||
# 新方案:Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
|
||||
from scipy.signal import savgol_filter
|
||||
try:
|
||||
# window_length 必须是奇数,取 7(或波段数的一半向下取奇数)
|
||||
_wl = min(7, n_features - 1)
|
||||
if _wl % 2 == 0:
|
||||
_wl -= 1
|
||||
if _wl >= 5:
|
||||
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
|
||||
noise = X - X_smooth
|
||||
else:
|
||||
# 波段太少,无法滤波,回退到去均值残差
|
||||
noise = X - np.mean(X, axis=1, keepdims=True)
|
||||
except Exception:
|
||||
# 兜底:去均值残差
|
||||
noise = X - np.mean(X, axis=1, keepdims=True)
|
||||
Cn = np.cov(noise, rowvar=False)
|
||||
_reg = np.trace(Cn) / n_features * 0.01
|
||||
Cn += np.eye(n_features) * max(_reg, 1e-6)
|
||||
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
|
||||
return IdentityTransformer()
|
||||
if method == "DualStream_MNF":
|
||||
from sklearn.pipeline import FeatureUnion
|
||||
return FeatureUnion([
|
||||
from sklearn.pipeline import Pipeline as _Pipeline
|
||||
# ★ 致命修复 2:MNF 主成分(量级 ~±10)与 Physical 指数(量级 ~0.001)
|
||||
# 尺度差异可达 10^4 倍,SVR 的 RBF 距离会被大值特征主导。
|
||||
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
|
||||
mnf_union = FeatureUnion([
|
||||
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
|
||||
('mnf', MNFTransformer(n_components=mnf_n_components)),
|
||||
])
|
||||
return _Pipeline([
|
||||
('features', mnf_union),
|
||||
('scaler', StandardScaler()),
|
||||
])
|
||||
if method == "Physical_Only":
|
||||
from sklearn.pipeline import Pipeline as _Pipeline
|
||||
return _Pipeline([
|
||||
|
||||
Reference in New Issue
Block a user