Compare commits

...

3 Commits

Author SHA1 Message Date
c6523cebd8 fix: GUI 模型加载改为按 metric 遍历所有 .joblib 选最优
旧行为: _scan_external_model_dir 盲目取 joblib_files[0]
        → 字母序选到 LinearRegression 而非 SVR
        → 所有目标永远是 D1,因为字母序 D1 < SNV

新行为:
  - 读取 GUI 中用户选择的 metric (R²/RMSE/MAE)
  - 遍历每个子目录下全部 .joblib 文件
  - 按指标自动选择最优模型:
    R²  → 取最大值
    RMSE → 取最小值
    MAE  → 取最小值
  - 控制台打印: [模型加载] 目标 BGA 根据 R² 自动选择最佳模型: BGA_spxy_SNV_SVR.joblib, 分数: 0.899001
2026-08-04 09:10:34 +08:00
04f9a647d8 fix: 推理端自适应反射率缩放 + Pipeline 列精确对齐
1. 自适应反射率量级缩放 (inference_pipeline/batch_inference/batch_inference_multi_data):
   - 检测光谱列 max > 10 时自动 ÷10000 统一到 0-1 区间
   - 三个推理入口全量同步保护

2. Pipeline 列精确对齐 (preprocess_spectra 两处):
   - 旧: not in 盲目排除多余列 → 140个光谱列被当多余移除
   - 新: 遍历 train_cols,精确字符串匹配 → math.isclose 浮点近似
        匹配列重命名为训练列名,缺失列补零
        最终 spectra[train_cols] 严格按训练顺序输出
   - 兼容路径同步修复

3. 误导性日志修正:
   - 改前: '正在应用预处理方法: D1' → 让用户以为推理端在手动做 D1
   - 改后: '[模型信息] 训练预处理方法: D1 — 由 Pipeline 内部自动执行'
2026-08-04 09:10:26 +08:00
f801b481fe fix: SVR+MNF 训练管线三个致命 Bug 修复
1. SVR y 尺度修复 (modeling_batch.py):
   - TransformedTargetRegressor 包裹 SVR,自动标准化 y 后训练
   - 超参前缀 model__ → model__regressor__ (穿透包裹层)
   - _find_svr_in_pipeline 兼容新的 TransformedTargetRegressor 嵌套

2. MNF 噪声估计修复 (spectral_Preprocessing.py):
   - 旧: noise = X[1:] - X[:-1] (假设空间连续,对打乱数据错误)
   - 新: noise = X - savgol_filter(X) (SG 滤波残差提取纯光谱噪声)
   - 附带波段数检查和异常兜底

3. DualStream_MNF 补标准 (spectral_Preprocessing.py):
   - FeatureUnion 外再包 Pipeline + StandardScaler
   - MNF 主成分(±10) 和 Physical 指数(0.001) 统一量纲后再送 SVR
2026-08-04 09:10:18 +08:00
4 changed files with 321 additions and 64 deletions

View File

@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import TransformedTargetRegressor
from sklearn.preprocessing import StandardScaler
from joblib import parallel_backend
# 第三方模型导入
# try:
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
elif model_name == 'LightGBM':
base_model.set_params(verbose=-1)
# ★ 致命修复 1:SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000),
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y),
# predict 时自动 inverse_transform,对调用方完全透明。
_is_svr = (config['model'] == SVR)
if _is_svr:
base_model = TransformedTargetRegressor(
regressor=base_model,
transformer=StandardScaler()
)
# ============ 关键:把预处理器塞进 Pipeline ============
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
_wl_list = None
@ -677,9 +690,16 @@ class WaterQualityModelingBatch:
# 以「步骤名__参数名」的格式索引参数网格;
# config['params'] 是模型层的(无 __),统一加 model__ 前缀。
prefixed_params = {
f"model__{k}": v for k, v in config['params'].items()
}
# ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
# GridSearchCV 参数路径变为 model__regressor__{param}
if _is_svr:
prefixed_params = {
f"model__regressor__{k}": v for k, v in config['params'].items()
}
else:
prefixed_params = {
f"model__{k}": v for k, v in config['params'].items()
}
# 全量网格搜索:SVR 超参组合仅 216 种(4×6×3×3),
# 穷举远优于 RandomizedSearchCV(n_iter=10) 的随机抽样
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
@staticmethod
def _find_svr_in_pipeline(pipeline):
model_step = pipeline.named_steps.get('model')
if model_step is not None and model_step.__class__.__name__ == 'SVR':
if model_step is None:
return None
# 直接就是 SVR(旧模型 / 非 SVR 的其他回归器)
if model_step.__class__.__name__ == 'SVR':
return model_step
# ★ TransformedTargetRegressor 包裹的 SVR(修复 1 引入)
if hasattr(model_step, 'regressor_'):
inner = model_step.regressor_
if inner.__class__.__name__ == 'SVR':
return inner
return None
@staticmethod

View File

@ -2,6 +2,7 @@ import numpy as np
import pandas as pd
import joblib
import os
import math
from pathlib import Path
from typing import List, Dict, Union, Tuple, Optional
import warnings
@ -546,12 +547,18 @@ class WaterQualityInference:
if actual_preprocess_method.lower() in ['nan', 'none', '']:
actual_preprocess_method = "None"
print(f"正在应用预处理方法: {actual_preprocess_method}")
print(f"原始光谱数据形状: {spectra.shape}")
model = self.loaded_model_data['model']
metadata = self.loaded_model_data.get('metadata', {})
# ★ 根据模型类型确定预处理归属
if isinstance(model, Pipeline):
print(f"[模型信息] 训练预处理方法: {actual_preprocess_method}"
f" — 由 Pipeline 内部自动执行,推理端仅做光谱重采样+列对齐")
else:
print(f"[模型信息] 训练预处理方法: {actual_preprocess_method}"
f" — 旧裸模型,推理端手动应用外部 Preprocessing")
print(f"原始光谱数据形状: {spectra.shape}")
# ═══════════════════════════════════════════════════════════
# ★ DualStream_MNF:纯光谱重采样 → pipeline.predict() 全自动
# ═══════════════════════════════════════════════════════════
@ -633,24 +640,67 @@ class WaterQualityInference:
# ── 兼容旧模型(无 train_wavelengths)──
print("[光谱重采样] 模型无 train_wavelengths 元数据(旧模型),"
"跳过光谱重采样,进入兼容路径...")
# 旧兼容逻辑:WQI 自动补全
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
# 旧兼容逻辑:WQI 自动补全(仅对旧裸模型生效;Pipeline 走 train_columns 对齐)
if isinstance(model, Pipeline):
train_cols = metadata.get('train_columns', None)
if train_cols is not None and len(train_cols) > 0:
print(f"[兼容+Pipeline 列对齐] 训练列数: {len(train_cols)}, 推理当前列数: {spectra.shape[1]}")
spectra_cols = [str(c) for c in spectra.columns]
mapping = {}
unmatched = []
for tc in train_cols:
tc_str = str(tc)
if tc_str in spectra_cols:
mapping[tc_str] = tc_str
else:
unmatched.append(tc_str)
still_unmatched = []
for tc_str in unmatched:
tc_float = None
try: tc_float = float(tc_str)
except (ValueError, TypeError): pass
if tc_float is not None:
found = None
for sc in spectra_cols:
if sc in mapping: continue
try:
if math.isclose(tc_float, float(sc), rel_tol=1e-4):
found = sc; break
except (ValueError, TypeError): pass
if found is not None:
mapping[found] = tc_str; continue
still_unmatched.append(tc_str)
aligned_parts = []
for sc in spectra_cols:
if sc in mapping:
col_data = spectra[sc].copy()
col_data.name = mapping[sc]
aligned_parts.append(col_data)
spectra_aligned = pd.concat(aligned_parts, axis=1) if aligned_parts else pd.DataFrame(index=spectra.index)
for tc_str in still_unmatched:
spectra_aligned[tc_str] = 0.0
spectra = spectra_aligned[[str(c) for c in train_cols]]
print(f"[兼容+Pipeline 列对齐] 完成 → {spectra.shape[1]} 列")
else:
print("[兼容+Pipeline] 无 train_columns,跳过 WQI 补全")
else:
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
# ★ v4: 无 train_wavelengths 时,基于物理波长的经验重采样
# 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格
@ -720,26 +770,97 @@ class WaterQualityInference:
f"{n_current} → {expected_features} 列")
# ═══════════════════════════════════════════════════════════
# ★ 特征补全:模型训练时可能包含 WQI 指数等衍生特征,
# 推理端需自动计算补齐(适用于新旧模型两条路径)
# ★ 特征列精确对齐:训练端可能包含 WQI 指数等衍生特征,
# 推理端必须精确匹配训练时的列集合,不能盲目补全。
# ═══════════════════════════════════════════════════════════
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
if isinstance(model, Pipeline):
# ── Pipeline 模型:用 train_columns 做精确列对齐 ──
train_cols = metadata.get('train_columns', None)
if train_cols is not None and len(train_cols) > 0:
print(f"[Pipeline 列对齐] 训练列数: {len(train_cols)}, 推理当前列数: {spectra.shape[1]}")
# 构建 推理列名 → 训练列名 的映射
spectra_cols = [str(c) for c in spectra.columns]
mapping: Dict[str, str] = {} # spectra_col_name → train_col_name
train_set = set(str(tc) for tc in train_cols)
# 第一遍:精确字符串匹配
unmatched_train = []
for tc in train_cols:
tc_str = str(tc)
if tc_str in spectra_cols:
mapping[tc_str] = tc_str
else:
unmatched_train.append(tc_str)
# 第二遍:对未匹配的训练列,尝试浮点数近似匹配
still_unmatched = []
for tc_str in unmatched_train:
tc_float = None
try:
tc_float = float(tc_str)
except (ValueError, TypeError):
pass
if tc_float is not None:
found = None
for sc in spectra_cols:
if sc in mapping: # 已经被匹配过了
continue
try:
sc_float = float(sc)
if math.isclose(tc_float, sc_float, rel_tol=1e-4):
found = sc
break
except (ValueError, TypeError):
pass
if found is not None:
mapping[found] = tc_str
continue
still_unmatched.append(tc_str)
# 从 spectra 中挑选已匹配的列,并重命名为训练列名
aligned_parts = []
matched_train = set(mapping.values())
for sc in spectra_cols:
if sc in mapping:
col_data = spectra[sc].copy()
col_data.name = mapping[sc] # 重命名为训练列名
aligned_parts.append(col_data)
spectra_aligned = pd.concat(aligned_parts, axis=1) if aligned_parts else pd.DataFrame(index=spectra.index)
# 对仍未匹配的训练列:WQI_ 补零,其他警告
for tc_str in still_unmatched:
spectra_aligned[tc_str] = 0.0
# 最终按 train_cols 顺序排列输出
spectra = spectra_aligned[[str(c) for c in train_cols]]
matched_count = len(matched_train)
missing_count = len(still_unmatched)
print(f"[Pipeline 列对齐] 精确匹配 {matched_count - missing_count} 列, "
f"浮点近似匹配 {len(mapping) - (matched_count - missing_count)} 列, "
f"补零填充 {missing_count} 列 → 最终 {spectra.shape[1]} 列")
else:
print("[Pipeline 列对齐] 无 train_columns 元数据,跳过列对齐")
else:
# ── 旧裸模型:保留原有 WQI 自动补全逻辑 ──
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
# ═══════════════════════════════════════════════════════════
# 通用清洗
@ -1009,6 +1130,33 @@ class WaterQualityInference:
print("-" * 40)
coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path)
# ═══════════════════════════════════════════════════════════════
# ★ 自适应反射率量级缩放 (Scale Alignment)
# ═══════════════════════════════════════════════════════════════
# 不同的高光谱传感器 / 处理流程产出的反射率量级可能不同:
# - float32 0-1 物理反射率(如 result3.bsq 抽样后写入的 CSV)
# - int16 0-10000 放大反射率(如 ref_mosaic 抽样后写入的 CSV)
# 若不经缩放直接喂入 SVR,量级差异会导致预测完全失效。
# 此处在光谱列上自动检测并统一到 0-1 区间。
spec_cols = []
for c in spectra.columns:
try:
float(str(c))
spec_cols.append(c)
except (ValueError, TypeError):
pass
if spec_cols:
max_val = spectra[spec_cols].max().max()
if max_val > 10:
print(f"\n[量级检测] 输入反射率疑似放大格式 (max={max_val:.2f})")
print("[量级检测] 自动除以 10000,缩放至 0-1 标准物理反射率区间...")
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
print(f"[量级检测] 缩放完成!缩放后 max={spectra[spec_cols].max().max():.4f}")
else:
print(f"[量级检测] 输入反射率量级正常 (max={max_val:.4f}),无需缩放")
# ═══════════════════════════════════════════════════════════════
# 3. 数据预处理
print("\n步骤3: 数据预处理")
print("-" * 40)
@ -1097,12 +1245,21 @@ class WaterQualityInference:
# 执行推理
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
# 自适应反射率量级缩放
_s_cols = []
for _c in spectra.columns:
try: float(str(_c)); _s_cols.append(_c)
except (ValueError, TypeError): pass
if _s_cols:
_mv = spectra[_s_cols].max().max()
if _mv > 10:
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
spectra_processed = self.preprocess_spectra(spectra)
predictions = self.predict(spectra_processed)
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
result_df = self.save_predictions(coords, predictions, str(output_file),
prediction_column, wqi_df)
results[csv_file.name] = {
'output_file': str(output_file),
'sample_count': len(predictions),
@ -1297,12 +1454,21 @@ class WaterQualityInference:
# 执行推理
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
# 自适应反射率量级缩放
_s_cols = []
for _c in spectra.columns:
try: float(str(_c)); _s_cols.append(_c)
except (ValueError, TypeError): pass
if _s_cols:
_mv = spectra[_s_cols].max().max()
if _mv > 10:
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
spectra_processed = self.preprocess_spectra(spectra)
predictions = self.predict(spectra_processed)
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
result_df = self.save_predictions(coords, predictions, str(output_file),
prediction_column, wqi_df)
results[file_stem] = {
'input_file': str(csv_file),
'output_file': str(output_file),

View File

@ -233,6 +233,14 @@ class Step9MlPredictPanel(QWidget):
return
self.external_model_dir = dir_path
# ★ 读取用户选择的模型评估指标
_metric_key = self.metric.currentData() # "test_r2" / "test_rmse" / "test_mae"
_metric_display = self.metric.currentText() # "R² (决定系数)" / ...
_is_r2 = 'r2' in _metric_key.lower()
_is_rmse = 'rmse' in _metric_key.lower()
_is_mae = 'mae' in _metric_key.lower()
models_found = {}
errors = []
@ -243,26 +251,53 @@ class Step9MlPredictPanel(QWidget):
if not subentry.is_dir():
continue
subdir_name = subentry.name
joblib_files = [
joblib_files = sorted([
f for f in os.scandir(subentry.path)
if f.is_file() and f.name.lower().endswith(".joblib")
]
], key=lambda x: x.name)
if not joblib_files:
continue
joblib_path = joblib_files[0].path
try:
loaded = joblib.load(joblib_path)
if isinstance(loaded, dict) and "model" in loaded:
# ★ 保留完整 dict(含 metadata / train_wavelengths),
# 推理端需要 train_wavelengths 做光谱重采样
models_found[subdir_name] = loaded
elif hasattr(loaded, "predict"):
models_found[subdir_name] = loaded
else:
errors.append(f"{subdir_name}: 无法识别的格式 {type(loaded).__name__}")
# ★ 遍历该子目录下全部 .joblib 文件,按 metric 选出最佳模型
best_score = -float('inf') if _is_r2 else float('inf')
best_entry = None
best_fname = None
for f_entry in joblib_files:
try:
data = joblib.load(f_entry.path)
if not isinstance(data, dict) or "model" not in data:
continue
meta = data.get('metadata', {})
# 按指标优先级:test_xxx → train_xxx
score = None
if _is_r2:
score = meta.get('test_r2', meta.get('train_r2', None))
elif _is_rmse:
score = meta.get('test_rmse', meta.get('train_rmse', None))
elif _is_mae:
score = meta.get('test_mae', meta.get('train_mae', None))
if score is None:
continue
if _is_r2 and score > best_score:
best_score = score
best_entry = data
best_fname = f_entry.name
elif not _is_r2 and score < best_score:
best_score = score
best_entry = data
best_fname = f_entry.name
except Exception:
continue
except Exception as e:
errors.append(f"{subdir_name}: {type(e).__name__}: {e}")
if best_entry is not None:
models_found[subdir_name] = best_entry
print(f"[模型加载] 目标 {subdir_name} 根据 {_metric_display} "
f"自动选择最佳模型: {best_fname}, 分数: {best_score:.6f}")
else:
errors.append(f"{subdir_name}: 无可评估的 .joblib 文件")
except Exception as e:
QMessageBox.warning(

View File

@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
n_samples, n_features = X.shape
# 1) 噪声矩阵:相邻行差分
noise = X[1:] - X[:-1]
# 1) ★ 致命修复 3:噪声矩阵估计。
# 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
# 白化操作会把真实水质信号当噪声除掉。
# 新方案:Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
from scipy.signal import savgol_filter
try:
# window_length 必须是奇数,取 7(或波段数的一半向下取奇数)
_wl = min(7, n_features - 1)
if _wl % 2 == 0:
_wl -= 1
if _wl >= 5:
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
noise = X - X_smooth
else:
# 波段太少,无法滤波,回退到去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
except Exception:
# 兜底:去均值残差
noise = X - np.mean(X, axis=1, keepdims=True)
Cn = np.cov(noise, rowvar=False)
_reg = np.trace(Cn) / n_features * 0.01
Cn += np.eye(n_features) * max(_reg, 1e-6)
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
return IdentityTransformer()
if method == "DualStream_MNF":
from sklearn.pipeline import FeatureUnion
return FeatureUnion([
from sklearn.pipeline import Pipeline as _Pipeline
# ★ 致命修复 2:MNF 主成分(量级 ~±10)与 Physical 指数(量级 ~0.001)
# 尺度差异可达 10^4 倍,SVR 的 RBF 距离会被大值特征主导。
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
mnf_union = FeatureUnion([
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
('mnf', MNFTransformer(n_components=mnf_n_components)),
])
return _Pipeline([
('features', mnf_union),
('scaler', StandardScaler()),
])
if method == "Physical_Only":
from sklearn.pipeline import Pipeline as _Pipeline
return _Pipeline([