fix: SVR+MNF 训练管线三个致命 Bug 修复
1. SVR y 尺度修复 (modeling_batch.py): - TransformedTargetRegressor 包裹 SVR,自动标准化 y 后训练 - 超参前缀 model__ → model__regressor__ (穿透包裹层) - _find_svr_in_pipeline 兼容新的 TransformedTargetRegressor 嵌套 2. MNF 噪声估计修复 (spectral_Preprocessing.py): - 旧: noise = X[1:] - X[:-1] (假设空间连续,对打乱数据错误) - 新: noise = X - savgol_filter(X) (SG 滤波残差提取纯光谱噪声) - 附带波段数检查和异常兜底 3. DualStream_MNF 补标准 (spectral_Preprocessing.py): - FeatureUnion 外再包 Pipeline + StandardScaler - MNF 主成分(±10) 和 Physical 指数(0.001) 统一量纲后再送 SVR
This commit is contained in:
@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
|
|||||||
from sklearn.neural_network import MLPRegressor
|
from sklearn.neural_network import MLPRegressor
|
||||||
from sklearn.pipeline import Pipeline
|
from sklearn.pipeline import Pipeline
|
||||||
from sklearn.impute import SimpleImputer
|
from sklearn.impute import SimpleImputer
|
||||||
|
from sklearn.compose import TransformedTargetRegressor
|
||||||
|
from sklearn.preprocessing import StandardScaler
|
||||||
from joblib import parallel_backend
|
from joblib import parallel_backend
|
||||||
# 第三方模型导入
|
# 第三方模型导入
|
||||||
# try:
|
# try:
|
||||||
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
|
|||||||
elif model_name == 'LightGBM':
|
elif model_name == 'LightGBM':
|
||||||
base_model.set_params(verbose=-1)
|
base_model.set_params(verbose=-1)
|
||||||
|
|
||||||
|
# ★ 致命修复 1:SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000),
|
||||||
|
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
|
||||||
|
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y),
|
||||||
|
# predict 时自动 inverse_transform,对调用方完全透明。
|
||||||
|
_is_svr = (config['model'] == SVR)
|
||||||
|
if _is_svr:
|
||||||
|
base_model = TransformedTargetRegressor(
|
||||||
|
regressor=base_model,
|
||||||
|
transformer=StandardScaler()
|
||||||
|
)
|
||||||
|
|
||||||
# ============ 关键:把预处理器塞进 Pipeline ============
|
# ============ 关键:把预处理器塞进 Pipeline ============
|
||||||
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
|
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
|
||||||
_wl_list = None
|
_wl_list = None
|
||||||
@ -677,6 +690,13 @@ class WaterQualityModelingBatch:
|
|||||||
|
|
||||||
# 以「步骤名__参数名」的格式索引参数网格;
|
# 以「步骤名__参数名」的格式索引参数网格;
|
||||||
# config['params'] 是模型层的(无 __),统一加 model__ 前缀。
|
# config['params'] 是模型层的(无 __),统一加 model__ 前缀。
|
||||||
|
# ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
|
||||||
|
# GridSearchCV 参数路径变为 model__regressor__{param}
|
||||||
|
if _is_svr:
|
||||||
|
prefixed_params = {
|
||||||
|
f"model__regressor__{k}": v for k, v in config['params'].items()
|
||||||
|
}
|
||||||
|
else:
|
||||||
prefixed_params = {
|
prefixed_params = {
|
||||||
f"model__{k}": v for k, v in config['params'].items()
|
f"model__{k}": v for k, v in config['params'].items()
|
||||||
}
|
}
|
||||||
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _find_svr_in_pipeline(pipeline):
|
def _find_svr_in_pipeline(pipeline):
|
||||||
model_step = pipeline.named_steps.get('model')
|
model_step = pipeline.named_steps.get('model')
|
||||||
if model_step is not None and model_step.__class__.__name__ == 'SVR':
|
if model_step is None:
|
||||||
|
return None
|
||||||
|
# 直接就是 SVR(旧模型 / 非 SVR 的其他回归器)
|
||||||
|
if model_step.__class__.__name__ == 'SVR':
|
||||||
return model_step
|
return model_step
|
||||||
|
# ★ TransformedTargetRegressor 包裹的 SVR(修复 1 引入)
|
||||||
|
if hasattr(model_step, 'regressor_'):
|
||||||
|
inner = model_step.regressor_
|
||||||
|
if inner.__class__.__name__ == 'SVR':
|
||||||
|
return inner
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
|||||||
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
|
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
|
||||||
n_samples, n_features = X.shape
|
n_samples, n_features = X.shape
|
||||||
|
|
||||||
# 1) 噪声矩阵:相邻行差分
|
# 1) ★ 致命修复 3:噪声矩阵估计。
|
||||||
noise = X[1:] - X[:-1]
|
# 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
|
||||||
|
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
|
||||||
|
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
|
||||||
|
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
|
||||||
|
# 白化操作会把真实水质信号当噪声除掉。
|
||||||
|
# 新方案:Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
|
||||||
|
from scipy.signal import savgol_filter
|
||||||
|
try:
|
||||||
|
# window_length 必须是奇数,取 7(或波段数的一半向下取奇数)
|
||||||
|
_wl = min(7, n_features - 1)
|
||||||
|
if _wl % 2 == 0:
|
||||||
|
_wl -= 1
|
||||||
|
if _wl >= 5:
|
||||||
|
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
|
||||||
|
noise = X - X_smooth
|
||||||
|
else:
|
||||||
|
# 波段太少,无法滤波,回退到去均值残差
|
||||||
|
noise = X - np.mean(X, axis=1, keepdims=True)
|
||||||
|
except Exception:
|
||||||
|
# 兜底:去均值残差
|
||||||
|
noise = X - np.mean(X, axis=1, keepdims=True)
|
||||||
Cn = np.cov(noise, rowvar=False)
|
Cn = np.cov(noise, rowvar=False)
|
||||||
_reg = np.trace(Cn) / n_features * 0.01
|
_reg = np.trace(Cn) / n_features * 0.01
|
||||||
Cn += np.eye(n_features) * max(_reg, 1e-6)
|
Cn += np.eye(n_features) * max(_reg, 1e-6)
|
||||||
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
|
|||||||
return IdentityTransformer()
|
return IdentityTransformer()
|
||||||
if method == "DualStream_MNF":
|
if method == "DualStream_MNF":
|
||||||
from sklearn.pipeline import FeatureUnion
|
from sklearn.pipeline import FeatureUnion
|
||||||
return FeatureUnion([
|
from sklearn.pipeline import Pipeline as _Pipeline
|
||||||
|
# ★ 致命修复 2:MNF 主成分(量级 ~±10)与 Physical 指数(量级 ~0.001)
|
||||||
|
# 尺度差异可达 10^4 倍,SVR 的 RBF 距离会被大值特征主导。
|
||||||
|
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
|
||||||
|
mnf_union = FeatureUnion([
|
||||||
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
|
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
|
||||||
('mnf', MNFTransformer(n_components=mnf_n_components)),
|
('mnf', MNFTransformer(n_components=mnf_n_components)),
|
||||||
])
|
])
|
||||||
|
return _Pipeline([
|
||||||
|
('features', mnf_union),
|
||||||
|
('scaler', StandardScaler()),
|
||||||
|
])
|
||||||
if method == "Physical_Only":
|
if method == "Physical_Only":
|
||||||
from sklearn.pipeline import Pipeline as _Pipeline
|
from sklearn.pipeline import Pipeline as _Pipeline
|
||||||
return _Pipeline([
|
return _Pipeline([
|
||||||
|
|||||||
Reference in New Issue
Block a user