fix: SVR+MNF 训练管线三个致命 Bug 修复

1. SVR y 尺度修复 (modeling_batch.py):
   - TransformedTargetRegressor 包裹 SVR,自动标准化 y 后训练
   - 超参前缀 model__ → model__regressor__ (穿透包裹层)
   - _find_svr_in_pipeline 兼容新的 TransformedTargetRegressor 嵌套

2. MNF 噪声估计修复 (spectral_Preprocessing.py):
   - 旧: noise = X[1:] - X[:-1] (假设空间连续,对打乱数据错误)
   - 新: noise = X - savgol_filter(X) (SG 滤波残差提取纯光谱噪声)
   - 附带波段数检查和异常兜底

3. DualStream_MNF 补标准 (spectral_Preprocessing.py):
   - FeatureUnion 外再包 Pipeline + StandardScaler
   - MNF 主成分(±10) 和 Physical 指数(0.001) 统一量纲后再送 SVR
This commit is contained in:
duxin
2026-08-04 09:10:18 +08:00
parent e3a08e0ad0
commit f801b481fe
2 changed files with 63 additions and 7 deletions

View File

@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import TransformedTargetRegressor
from sklearn.preprocessing import StandardScaler
from joblib import parallel_backend
# 第三方模型导入
# try:
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
elif model_name == 'LightGBM':
base_model.set_params(verbose=-1)
# ★ 致命修复 1SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y)
# predict 时自动 inverse_transform对调用方完全透明。
_is_svr = (config['model'] == SVR)
if _is_svr:
base_model = TransformedTargetRegressor(
regressor=base_model,
transformer=StandardScaler()
)
# ============ 关键:把预处理器塞进 Pipeline ============
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
_wl_list = None
@ -677,9 +690,16 @@ class WaterQualityModelingBatch:
# 以「步骤名__参数名」的格式索引参数网格
# config['params'] 是模型层的(无 __统一加 model__ 前缀。
prefixed_params = {
f"model__{k}": v for k, v in config['params'].items()
}
# ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
# GridSearchCV 参数路径变为 model__regressor__{param}
if _is_svr:
prefixed_params = {
f"model__regressor__{k}": v for k, v in config['params'].items()
}
else:
prefixed_params = {
f"model__{k}": v for k, v in config['params'].items()
}
# 全量网格搜索SVR 超参组合仅 216 种4×6×3×3
# 穷举远优于 RandomizedSearchCV(n_iter=10) 的随机抽样
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
@staticmethod
def _find_svr_in_pipeline(pipeline):
model_step = pipeline.named_steps.get('model')
if model_step is not None and model_step.__class__.__name__ == 'SVR':
if model_step is None:
return None
# 直接就是 SVR旧模型 / 非 SVR 的其他回归器)
if model_step.__class__.__name__ == 'SVR':
return model_step
# ★ TransformedTargetRegressor 包裹的 SVR修复 1 引入)
if hasattr(model_step, 'regressor_'):
inner = model_step.regressor_
if inner.__class__.__name__ == 'SVR':
return inner
return None
@staticmethod

View File

@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
n_samples, n_features = X.shape
# 1) 噪声矩阵:相邻行差分
noise = X[1:] - X[:-1]
# 1) ★ 致命修复 3噪声矩阵估计。
# 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
# 白化操作会把真实水质信号当噪声除掉。
# 新方案Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
from scipy.signal import savgol_filter
try:
# window_length 必须是奇数,取 7或波段数的一半向下取奇数
_wl = min(7, n_features - 1)
if _wl % 2 == 0:
_wl -= 1
if _wl >= 5:
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
noise = X - X_smooth
else:
# 波段太少,无法滤波,回退到去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
except Exception:
# 兜底:去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
Cn = np.cov(noise, rowvar=False)
_reg = np.trace(Cn) / n_features * 0.01
Cn += np.eye(n_features) * max(_reg, 1e-6)
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
return IdentityTransformer()
if method == "DualStream_MNF":
from sklearn.pipeline import FeatureUnion
return FeatureUnion([
from sklearn.pipeline import Pipeline as _Pipeline
# ★ 致命修复 2MNF 主成分(量级 ~±10与 Physical 指数(量级 ~0.001
# 尺度差异可达 10^4 倍SVR 的 RBF 距离会被大值特征主导。
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
mnf_union = FeatureUnion([
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
('mnf', MNFTransformer(n_components=mnf_n_components)),
])
return _Pipeline([
('features', mnf_union),
('scaler', StandardScaler()),
])
if method == "Physical_Only":
from sklearn.pipeline import Pipeline as _Pipeline
return _Pipeline([