fix: SVR+MNF 训练管线三个致命 Bug 修复

1. SVR y 尺度修复 (modeling_batch.py):
   - TransformedTargetRegressor 包裹 SVR,自动标准化 y 后训练
   - 超参前缀 model__ → model__regressor__ (穿透包裹层)
   - _find_svr_in_pipeline 兼容新的 TransformedTargetRegressor 嵌套

2. MNF 噪声估计修复 (spectral_Preprocessing.py):
   - 旧: noise = X[1:] - X[:-1] (假设空间连续,对打乱数据错误)
   - 新: noise = X - savgol_filter(X) (SG 滤波残差提取纯光谱噪声)
   - 附带波段数检查和异常兜底

3. DualStream_MNF 补标准 (spectral_Preprocessing.py):
   - FeatureUnion 外再包 Pipeline + StandardScaler
   - MNF 主成分(±10) 和 Physical 指数(0.001) 统一量纲后再送 SVR
This commit is contained in:
duxin
2026-08-04 09:10:18 +08:00
parent e3a08e0ad0
commit f801b481fe
2 changed files with 63 additions and 7 deletions

View File

@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
from sklearn.neural_network import MLPRegressor from sklearn.neural_network import MLPRegressor
from sklearn.pipeline import Pipeline from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer from sklearn.impute import SimpleImputer
from sklearn.compose import TransformedTargetRegressor
from sklearn.preprocessing import StandardScaler
from joblib import parallel_backend from joblib import parallel_backend
# 第三方模型导入 # 第三方模型导入
# try: # try:
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
elif model_name == 'LightGBM': elif model_name == 'LightGBM':
base_model.set_params(verbose=-1) base_model.set_params(verbose=-1)
# ★ 致命修复 1SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y)
# predict 时自动 inverse_transform对调用方完全透明。
_is_svr = (config['model'] == SVR)
if _is_svr:
base_model = TransformedTargetRegressor(
regressor=base_model,
transformer=StandardScaler()
)
# ============ 关键:把预处理器塞进 Pipeline ============ # ============ 关键:把预处理器塞进 Pipeline ============
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段 # DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
_wl_list = None _wl_list = None
@ -677,6 +690,13 @@ class WaterQualityModelingBatch:
# 以「步骤名__参数名」的格式索引参数网格 # 以「步骤名__参数名」的格式索引参数网格
# config['params'] 是模型层的(无 __统一加 model__ 前缀。 # config['params'] 是模型层的(无 __统一加 model__ 前缀。
# ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
# GridSearchCV 参数路径变为 model__regressor__{param}
if _is_svr:
prefixed_params = {
f"model__regressor__{k}": v for k, v in config['params'].items()
}
else:
prefixed_params = { prefixed_params = {
f"model__{k}": v for k, v in config['params'].items() f"model__{k}": v for k, v in config['params'].items()
} }
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
@staticmethod @staticmethod
def _find_svr_in_pipeline(pipeline): def _find_svr_in_pipeline(pipeline):
model_step = pipeline.named_steps.get('model') model_step = pipeline.named_steps.get('model')
if model_step is not None and model_step.__class__.__name__ == 'SVR': if model_step is None:
return None
# 直接就是 SVR旧模型 / 非 SVR 的其他回归器)
if model_step.__class__.__name__ == 'SVR':
return model_step return model_step
# ★ TransformedTargetRegressor 包裹的 SVR修复 1 引入)
if hasattr(model_step, 'regressor_'):
inner = model_step.regressor_
if inner.__class__.__name__ == 'SVR':
return inner
return None return None
@staticmethod @staticmethod

View File

@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0) X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
n_samples, n_features = X.shape n_samples, n_features = X.shape
# 1) 噪声矩阵:相邻行差分 # 1) ★ 致命修复 3噪声矩阵估计。
noise = X[1:] - X[:-1] # 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
# 白化操作会把真实水质信号当噪声除掉。
# 新方案Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
from scipy.signal import savgol_filter
try:
# window_length 必须是奇数,取 7或波段数的一半向下取奇数
_wl = min(7, n_features - 1)
if _wl % 2 == 0:
_wl -= 1
if _wl >= 5:
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
noise = X - X_smooth
else:
# 波段太少,无法滤波,回退到去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
except Exception:
# 兜底:去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
Cn = np.cov(noise, rowvar=False) Cn = np.cov(noise, rowvar=False)
_reg = np.trace(Cn) / n_features * 0.01 _reg = np.trace(Cn) / n_features * 0.01
Cn += np.eye(n_features) * max(_reg, 1e-6) Cn += np.eye(n_features) * max(_reg, 1e-6)
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
return IdentityTransformer() return IdentityTransformer()
if method == "DualStream_MNF": if method == "DualStream_MNF":
from sklearn.pipeline import FeatureUnion from sklearn.pipeline import FeatureUnion
return FeatureUnion([ from sklearn.pipeline import Pipeline as _Pipeline
# ★ 致命修复 2MNF 主成分(量级 ~±10与 Physical 指数(量级 ~0.001
# 尺度差异可达 10^4 倍SVR 的 RBF 距离会被大值特征主导。
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
mnf_union = FeatureUnion([
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)), ('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
('mnf', MNFTransformer(n_components=mnf_n_components)), ('mnf', MNFTransformer(n_components=mnf_n_components)),
]) ])
return _Pipeline([
('features', mnf_union),
('scaler', StandardScaler()),
])
if method == "Physical_Only": if method == "Physical_Only":
from sklearn.pipeline import Pipeline as _Pipeline from sklearn.pipeline import Pipeline as _Pipeline
return _Pipeline([ return _Pipeline([