Compare commits

..

3 Commits

Author SHA1 Message Date
c6523cebd8 fix: GUI 模型加载改为按 metric 遍历所有 .joblib 选最优
旧行为: _scan_external_model_dir 盲目取 joblib_files[0]
        → 字母序选到 LinearRegression 而非 SVR
        → 所有目标永远是 D1,因为字母序 D1 < SNV

新行为:
  - 读取 GUI 中用户选择的 metric (R²/RMSE/MAE)
  - 遍历每个子目录下全部 .joblib 文件
  - 按指标自动选择最优模型:
    R²  → 取最大值
    RMSE → 取最小值
    MAE  → 取最小值
  - 控制台打印: [模型加载] 目标 BGA 根据 R² 自动选择最佳模型: BGA_spxy_SNV_SVR.joblib, 分数: 0.899001
2026-08-04 09:10:34 +08:00
04f9a647d8 fix: 推理端自适应反射率缩放 + Pipeline 列精确对齐
1. 自适应反射率量级缩放 (inference_pipeline/batch_inference/batch_inference_multi_data):
   - 检测光谱列 max > 10 时自动 ÷10000 统一到 0-1 区间
   - 三个推理入口全量同步保护

2. Pipeline 列精确对齐 (preprocess_spectra 两处):
   - 旧: not in 盲目排除多余列 → 140个光谱列被当多余移除
   - 新: 遍历 train_cols,精确字符串匹配 → math.isclose 浮点近似
        匹配列重命名为训练列名,缺失列补零
        最终 spectra[train_cols] 严格按训练顺序输出
   - 兼容路径同步修复

3. 误导性日志修正:
   - 改前: '正在应用预处理方法: D1' → 让用户以为推理端在手动做 D1
   - 改后: '[模型信息] 训练预处理方法: D1 — 由 Pipeline 内部自动执行'
2026-08-04 09:10:26 +08:00
f801b481fe fix: SVR+MNF 训练管线三个致命 Bug 修复
1. SVR y 尺度修复 (modeling_batch.py):
   - TransformedTargetRegressor 包裹 SVR,自动标准化 y 后训练
   - 超参前缀 model__ → model__regressor__ (穿透包裹层)
   - _find_svr_in_pipeline 兼容新的 TransformedTargetRegressor 嵌套

2. MNF 噪声估计修复 (spectral_Preprocessing.py):
   - 旧: noise = X[1:] - X[:-1] (假设空间连续,对打乱数据错误)
   - 新: noise = X - savgol_filter(X) (SG 滤波残差提取纯光谱噪声)
   - 附带波段数检查和异常兜底

3. DualStream_MNF 补标准 (spectral_Preprocessing.py):
   - FeatureUnion 外再包 Pipeline + StandardScaler
   - MNF 主成分(±10) 和 Physical 指数(0.001) 统一量纲后再送 SVR
2026-08-04 09:10:18 +08:00
4 changed files with 321 additions and 64 deletions

View File

@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
from sklearn.neural_network import MLPRegressor from sklearn.neural_network import MLPRegressor
from sklearn.pipeline import Pipeline from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer from sklearn.impute import SimpleImputer
from sklearn.compose import TransformedTargetRegressor
from sklearn.preprocessing import StandardScaler
from joblib import parallel_backend from joblib import parallel_backend
# 第三方模型导入 # 第三方模型导入
# try: # try:
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
elif model_name == 'LightGBM': elif model_name == 'LightGBM':
base_model.set_params(verbose=-1) base_model.set_params(verbose=-1)
# ★ 致命修复 1:SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000),
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y),
# predict 时自动 inverse_transform,对调用方完全透明。
_is_svr = (config['model'] == SVR)
if _is_svr:
base_model = TransformedTargetRegressor(
regressor=base_model,
transformer=StandardScaler()
)
# ============ 关键:把预处理器塞进 Pipeline ============ # ============ 关键:把预处理器塞进 Pipeline ============
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段 # DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
_wl_list = None _wl_list = None
@ -677,9 +690,16 @@ class WaterQualityModelingBatch:
# 以「步骤名__参数名」的格式索引参数网格; # 以「步骤名__参数名」的格式索引参数网格;
# config['params'] 是模型层的(无 __),统一加 model__ 前缀。 # config['params'] 是模型层的(无 __),统一加 model__ 前缀。
prefixed_params = { # ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
f"model__{k}": v for k, v in config['params'].items() # GridSearchCV 参数路径变为 model__regressor__{param}
} if _is_svr:
prefixed_params = {
f"model__regressor__{k}": v for k, v in config['params'].items()
}
else:
prefixed_params = {
f"model__{k}": v for k, v in config['params'].items()
}
# 全量网格搜索:SVR 超参组合仅 216 种(4×6×3×3), # 全量网格搜索:SVR 超参组合仅 216 种(4×6×3×3),
# 穷举远优于 RandomizedSearchCV(n_iter=10) 的随机抽样 # 穷举远优于 RandomizedSearchCV(n_iter=10) 的随机抽样
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
@staticmethod @staticmethod
def _find_svr_in_pipeline(pipeline): def _find_svr_in_pipeline(pipeline):
model_step = pipeline.named_steps.get('model') model_step = pipeline.named_steps.get('model')
if model_step is not None and model_step.__class__.__name__ == 'SVR': if model_step is None:
return None
# 直接就是 SVR(旧模型 / 非 SVR 的其他回归器)
if model_step.__class__.__name__ == 'SVR':
return model_step return model_step
# ★ TransformedTargetRegressor 包裹的 SVR(修复 1 引入)
if hasattr(model_step, 'regressor_'):
inner = model_step.regressor_
if inner.__class__.__name__ == 'SVR':
return inner
return None return None
@staticmethod @staticmethod

View File

@ -2,6 +2,7 @@ import numpy as np
import pandas as pd import pandas as pd
import joblib import joblib
import os import os
import math
from pathlib import Path from pathlib import Path
from typing import List, Dict, Union, Tuple, Optional from typing import List, Dict, Union, Tuple, Optional
import warnings import warnings
@ -546,12 +547,18 @@ class WaterQualityInference:
if actual_preprocess_method.lower() in ['nan', 'none', '']: if actual_preprocess_method.lower() in ['nan', 'none', '']:
actual_preprocess_method = "None" actual_preprocess_method = "None"
print(f"正在应用预处理方法: {actual_preprocess_method}")
print(f"原始光谱数据形状: {spectra.shape}")
model = self.loaded_model_data['model'] model = self.loaded_model_data['model']
metadata = self.loaded_model_data.get('metadata', {}) metadata = self.loaded_model_data.get('metadata', {})
# ★ 根据模型类型确定预处理归属
if isinstance(model, Pipeline):
print(f"[模型信息] 训练预处理方法: {actual_preprocess_method}"
f" — 由 Pipeline 内部自动执行,推理端仅做光谱重采样+列对齐")
else:
print(f"[模型信息] 训练预处理方法: {actual_preprocess_method}"
f" — 旧裸模型,推理端手动应用外部 Preprocessing")
print(f"原始光谱数据形状: {spectra.shape}")
# ═══════════════════════════════════════════════════════════ # ═══════════════════════════════════════════════════════════
# ★ DualStream_MNF:纯光谱重采样 → pipeline.predict() 全自动 # ★ DualStream_MNF:纯光谱重采样 → pipeline.predict() 全自动
# ═══════════════════════════════════════════════════════════ # ═══════════════════════════════════════════════════════════
@ -633,24 +640,67 @@ class WaterQualityInference:
# ── 兼容旧模型(无 train_wavelengths)── # ── 兼容旧模型(无 train_wavelengths)──
print("[光谱重采样] 模型无 train_wavelengths 元数据(旧模型)," print("[光谱重采样] 模型无 train_wavelengths 元数据(旧模型),"
"跳过光谱重采样,进入兼容路径...") "跳过光谱重采样,进入兼容路径...")
# 旧兼容逻辑:WQI 自动补全 # 旧兼容逻辑:WQI 自动补全(仅对旧裸模型生效;Pipeline 走 train_columns 对齐)
expected_features = getattr(model, 'n_features_in_', None) if isinstance(model, Pipeline):
if expected_features is not None and spectra.shape[1] < expected_features: train_cols = metadata.get('train_columns', None)
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 " if train_cols is not None and len(train_cols) > 0:
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...") print(f"[兼容+Pipeline 列对齐] 训练列数: {len(train_cols)}, 推理当前列数: {spectra.shape[1]}")
try: spectra_cols = [str(c) for c in spectra.columns]
from src.utils.water_index import WaterQualityIndexCalculator mapping = {}
calc = WaterQualityIndexCalculator() unmatched = []
formulas = calc.list_available() for tc in train_cols:
if formulas: tc_str = str(tc)
results_df = calc.calculate_many(formulas, spectra, fast=True) if tc_str in spectra_cols:
if isinstance(results_df, pd.DataFrame) and not results_df.empty: mapping[tc_str] = tc_str
original_col_count = spectra.shape[1] else:
spectra = pd.concat([spectra, results_df], axis=1) unmatched.append(tc_str)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 " still_unmatched = []
f"(+{spectra.shape[1] - original_col_count} WQI)") for tc_str in unmatched:
except Exception as e: tc_float = None
print(f"[特征补全] 失败: {e}") try: tc_float = float(tc_str)
except (ValueError, TypeError): pass
if tc_float is not None:
found = None
for sc in spectra_cols:
if sc in mapping: continue
try:
if math.isclose(tc_float, float(sc), rel_tol=1e-4):
found = sc; break
except (ValueError, TypeError): pass
if found is not None:
mapping[found] = tc_str; continue
still_unmatched.append(tc_str)
aligned_parts = []
for sc in spectra_cols:
if sc in mapping:
col_data = spectra[sc].copy()
col_data.name = mapping[sc]
aligned_parts.append(col_data)
spectra_aligned = pd.concat(aligned_parts, axis=1) if aligned_parts else pd.DataFrame(index=spectra.index)
for tc_str in still_unmatched:
spectra_aligned[tc_str] = 0.0
spectra = spectra_aligned[[str(c) for c in train_cols]]
print(f"[兼容+Pipeline 列对齐] 完成 → {spectra.shape[1]} 列")
else:
print("[兼容+Pipeline] 无 train_columns,跳过 WQI 补全")
else:
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
# ★ v4: 无 train_wavelengths 时,基于物理波长的经验重采样 # ★ v4: 无 train_wavelengths 时,基于物理波长的经验重采样
# 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格 # 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格
@ -720,26 +770,97 @@ class WaterQualityInference:
f"{n_current} → {expected_features} 列") f"{n_current} → {expected_features} 列")
# ═══════════════════════════════════════════════════════════ # ═══════════════════════════════════════════════════════════
# ★ 特征补全:模型训练时可能包含 WQI 指数等衍生特征, # ★ 特征列精确对齐:训练端可能包含 WQI 指数等衍生特征,
# 推理端需自动计算补齐(适用于新旧模型两条路径) # 推理端必须精确匹配训练时的列集合,不能盲目补全。
# ═══════════════════════════════════════════════════════════ # ═══════════════════════════════════════════════════════════
expected_features = getattr(model, 'n_features_in_', None) if isinstance(model, Pipeline):
if expected_features is not None and spectra.shape[1] < expected_features: # ── Pipeline 模型:用 train_columns 做精确列对齐 ──
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 " train_cols = metadata.get('train_columns', None)
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...") if train_cols is not None and len(train_cols) > 0:
try: print(f"[Pipeline 列对齐] 训练列数: {len(train_cols)}, 推理当前列数: {spectra.shape[1]}")
from src.utils.water_index import WaterQualityIndexCalculator # 构建 推理列名 → 训练列名 的映射
calc = WaterQualityIndexCalculator() spectra_cols = [str(c) for c in spectra.columns]
formulas = calc.list_available() mapping: Dict[str, str] = {} # spectra_col_name → train_col_name
if formulas: train_set = set(str(tc) for tc in train_cols)
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty: # 第一遍:精确字符串匹配
original_col_count = spectra.shape[1] unmatched_train = []
spectra = pd.concat([spectra, results_df], axis=1) for tc in train_cols:
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 " tc_str = str(tc)
f"(+{spectra.shape[1] - original_col_count} WQI)") if tc_str in spectra_cols:
except Exception as e: mapping[tc_str] = tc_str
print(f"[特征补全] 失败: {e}") else:
unmatched_train.append(tc_str)
# 第二遍:对未匹配的训练列,尝试浮点数近似匹配
still_unmatched = []
for tc_str in unmatched_train:
tc_float = None
try:
tc_float = float(tc_str)
except (ValueError, TypeError):
pass
if tc_float is not None:
found = None
for sc in spectra_cols:
if sc in mapping: # 已经被匹配过了
continue
try:
sc_float = float(sc)
if math.isclose(tc_float, sc_float, rel_tol=1e-4):
found = sc
break
except (ValueError, TypeError):
pass
if found is not None:
mapping[found] = tc_str
continue
still_unmatched.append(tc_str)
# 从 spectra 中挑选已匹配的列,并重命名为训练列名
aligned_parts = []
matched_train = set(mapping.values())
for sc in spectra_cols:
if sc in mapping:
col_data = spectra[sc].copy()
col_data.name = mapping[sc] # 重命名为训练列名
aligned_parts.append(col_data)
spectra_aligned = pd.concat(aligned_parts, axis=1) if aligned_parts else pd.DataFrame(index=spectra.index)
# 对仍未匹配的训练列:WQI_ 补零,其他警告
for tc_str in still_unmatched:
spectra_aligned[tc_str] = 0.0
# 最终按 train_cols 顺序排列输出
spectra = spectra_aligned[[str(c) for c in train_cols]]
matched_count = len(matched_train)
missing_count = len(still_unmatched)
print(f"[Pipeline 列对齐] 精确匹配 {matched_count - missing_count} 列, "
f"浮点近似匹配 {len(mapping) - (matched_count - missing_count)} 列, "
f"补零填充 {missing_count} 列 → 最终 {spectra.shape[1]} 列")
else:
print("[Pipeline 列对齐] 无 train_columns 元数据,跳过列对齐")
else:
# ── 旧裸模型:保留原有 WQI 自动补全逻辑 ──
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
# ═══════════════════════════════════════════════════════════ # ═══════════════════════════════════════════════════════════
# 通用清洗 # 通用清洗
@ -1009,6 +1130,33 @@ class WaterQualityInference:
print("-" * 40) print("-" * 40)
coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path) coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path)
# ═══════════════════════════════════════════════════════════════
# ★ 自适应反射率量级缩放 (Scale Alignment)
# ═══════════════════════════════════════════════════════════════
# 不同的高光谱传感器 / 处理流程产出的反射率量级可能不同:
# - float32 0-1 物理反射率(如 result3.bsq 抽样后写入的 CSV)
# - int16 0-10000 放大反射率(如 ref_mosaic 抽样后写入的 CSV)
# 若不经缩放直接喂入 SVR,量级差异会导致预测完全失效。
# 此处在光谱列上自动检测并统一到 0-1 区间。
spec_cols = []
for c in spectra.columns:
try:
float(str(c))
spec_cols.append(c)
except (ValueError, TypeError):
pass
if spec_cols:
max_val = spectra[spec_cols].max().max()
if max_val > 10:
print(f"\n[量级检测] 输入反射率疑似放大格式 (max={max_val:.2f})")
print("[量级检测] 自动除以 10000,缩放至 0-1 标准物理反射率区间...")
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
print(f"[量级检测] 缩放完成!缩放后 max={spectra[spec_cols].max().max():.4f}")
else:
print(f"[量级检测] 输入反射率量级正常 (max={max_val:.4f}),无需缩放")
# ═══════════════════════════════════════════════════════════════
# 3. 数据预处理 # 3. 数据预处理
print("\n步骤3: 数据预处理") print("\n步骤3: 数据预处理")
print("-" * 40) print("-" * 40)
@ -1097,12 +1245,21 @@ class WaterQualityInference:
# 执行推理 # 执行推理
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file)) coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
# 自适应反射率量级缩放
_s_cols = []
for _c in spectra.columns:
try: float(str(_c)); _s_cols.append(_c)
except (ValueError, TypeError): pass
if _s_cols:
_mv = spectra[_s_cols].max().max()
if _mv > 10:
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
spectra_processed = self.preprocess_spectra(spectra) spectra_processed = self.preprocess_spectra(spectra)
predictions = self.predict(spectra_processed) predictions = self.predict(spectra_processed)
predictions = self._mask_zero_spectra_pixels(spectra, predictions) predictions = self._mask_zero_spectra_pixels(spectra, predictions)
result_df = self.save_predictions(coords, predictions, str(output_file), result_df = self.save_predictions(coords, predictions, str(output_file),
prediction_column, wqi_df) prediction_column, wqi_df)
results[csv_file.name] = { results[csv_file.name] = {
'output_file': str(output_file), 'output_file': str(output_file),
'sample_count': len(predictions), 'sample_count': len(predictions),
@ -1297,12 +1454,21 @@ class WaterQualityInference:
# 执行推理 # 执行推理
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file)) coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
# 自适应反射率量级缩放
_s_cols = []
for _c in spectra.columns:
try: float(str(_c)); _s_cols.append(_c)
except (ValueError, TypeError): pass
if _s_cols:
_mv = spectra[_s_cols].max().max()
if _mv > 10:
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
spectra_processed = self.preprocess_spectra(spectra) spectra_processed = self.preprocess_spectra(spectra)
predictions = self.predict(spectra_processed) predictions = self.predict(spectra_processed)
predictions = self._mask_zero_spectra_pixels(spectra, predictions) predictions = self._mask_zero_spectra_pixels(spectra, predictions)
result_df = self.save_predictions(coords, predictions, str(output_file), result_df = self.save_predictions(coords, predictions, str(output_file),
prediction_column, wqi_df) prediction_column, wqi_df)
results[file_stem] = { results[file_stem] = {
'input_file': str(csv_file), 'input_file': str(csv_file),
'output_file': str(output_file), 'output_file': str(output_file),

View File

@ -233,6 +233,14 @@ class Step9MlPredictPanel(QWidget):
return return
self.external_model_dir = dir_path self.external_model_dir = dir_path
# ★ 读取用户选择的模型评估指标
_metric_key = self.metric.currentData() # "test_r2" / "test_rmse" / "test_mae"
_metric_display = self.metric.currentText() # "R² (决定系数)" / ...
_is_r2 = 'r2' in _metric_key.lower()
_is_rmse = 'rmse' in _metric_key.lower()
_is_mae = 'mae' in _metric_key.lower()
models_found = {} models_found = {}
errors = [] errors = []
@ -243,26 +251,53 @@ class Step9MlPredictPanel(QWidget):
if not subentry.is_dir(): if not subentry.is_dir():
continue continue
subdir_name = subentry.name subdir_name = subentry.name
joblib_files = [ joblib_files = sorted([
f for f in os.scandir(subentry.path) f for f in os.scandir(subentry.path)
if f.is_file() and f.name.lower().endswith(".joblib") if f.is_file() and f.name.lower().endswith(".joblib")
] ], key=lambda x: x.name)
if not joblib_files: if not joblib_files:
continue continue
joblib_path = joblib_files[0].path
try: # ★ 遍历该子目录下全部 .joblib 文件,按 metric 选出最佳模型
loaded = joblib.load(joblib_path) best_score = -float('inf') if _is_r2 else float('inf')
if isinstance(loaded, dict) and "model" in loaded: best_entry = None
# ★ 保留完整 dict(含 metadata / train_wavelengths), best_fname = None
# 推理端需要 train_wavelengths 做光谱重采样
models_found[subdir_name] = loaded for f_entry in joblib_files:
elif hasattr(loaded, "predict"): try:
models_found[subdir_name] = loaded data = joblib.load(f_entry.path)
else: if not isinstance(data, dict) or "model" not in data:
errors.append(f"{subdir_name}: 无法识别的格式 {type(loaded).__name__}") continue
meta = data.get('metadata', {})
# 按指标优先级:test_xxx → train_xxx
score = None
if _is_r2:
score = meta.get('test_r2', meta.get('train_r2', None))
elif _is_rmse:
score = meta.get('test_rmse', meta.get('train_rmse', None))
elif _is_mae:
score = meta.get('test_mae', meta.get('train_mae', None))
if score is None:
continue
if _is_r2 and score > best_score:
best_score = score
best_entry = data
best_fname = f_entry.name
elif not _is_r2 and score < best_score:
best_score = score
best_entry = data
best_fname = f_entry.name
except Exception:
continue continue
except Exception as e:
errors.append(f"{subdir_name}: {type(e).__name__}: {e}") if best_entry is not None:
models_found[subdir_name] = best_entry
print(f"[模型加载] 目标 {subdir_name} 根据 {_metric_display} "
f"自动选择最佳模型: {best_fname}, 分数: {best_score:.6f}")
else:
errors.append(f"{subdir_name}: 无可评估的 .joblib 文件")
except Exception as e: except Exception as e:
QMessageBox.warning( QMessageBox.warning(

View File

@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0) X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
n_samples, n_features = X.shape n_samples, n_features = X.shape
# 1) 噪声矩阵:相邻行差分 # 1) ★ 致命修复 3:噪声矩阵估计。
noise = X[1:] - X[:-1] # 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
# 白化操作会把真实水质信号当噪声除掉。
# 新方案:Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
from scipy.signal import savgol_filter
try:
# window_length 必须是奇数,取 7(或波段数的一半向下取奇数)
_wl = min(7, n_features - 1)
if _wl % 2 == 0:
_wl -= 1
if _wl >= 5:
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
noise = X - X_smooth
else:
# 波段太少,无法滤波,回退到去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
except Exception:
# 兜底:去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
Cn = np.cov(noise, rowvar=False) Cn = np.cov(noise, rowvar=False)
_reg = np.trace(Cn) / n_features * 0.01 _reg = np.trace(Cn) / n_features * 0.01
Cn += np.eye(n_features) * max(_reg, 1e-6) Cn += np.eye(n_features) * max(_reg, 1e-6)
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
return IdentityTransformer() return IdentityTransformer()
if method == "DualStream_MNF": if method == "DualStream_MNF":
from sklearn.pipeline import FeatureUnion from sklearn.pipeline import FeatureUnion
return FeatureUnion([ from sklearn.pipeline import Pipeline as _Pipeline
# ★ 致命修复 2:MNF 主成分(量级 ~±10)与 Physical 指数(量级 ~0.001)
# 尺度差异可达 10^4 倍,SVR 的 RBF 距离会被大值特征主导。
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
mnf_union = FeatureUnion([
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)), ('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
('mnf', MNFTransformer(n_components=mnf_n_components)), ('mnf', MNFTransformer(n_components=mnf_n_components)),
]) ])
return _Pipeline([
('features', mnf_union),
('scaler', StandardScaler()),
])
if method == "Physical_Only": if method == "Physical_Only":
from sklearn.pipeline import Pipeline as _Pipeline from sklearn.pipeline import Pipeline as _Pipeline
return _Pipeline([ return _Pipeline([