Compare commits
3 Commits
e3a08e0ad0
...
c6523cebd8
| Author | SHA1 | Date | |
|---|---|---|---|
| c6523cebd8 | |||
| 04f9a647d8 | |||
| f801b481fe |
@ -22,6 +22,8 @@ from sklearn.tree import DecisionTreeRegressor
|
||||
from sklearn.neural_network import MLPRegressor
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.compose import TransformedTargetRegressor
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from joblib import parallel_backend
|
||||
# 第三方模型导入
|
||||
# try:
|
||||
@ -655,6 +657,17 @@ class WaterQualityModelingBatch:
|
||||
elif model_name == 'LightGBM':
|
||||
base_model.set_params(verbose=-1)
|
||||
|
||||
# ★ 致命修复 1:SVR 对 y 尺度极度敏感。若 y 未缩放(如浊度 100~1000),
|
||||
# epsilon=0.1 相对 y 量级几乎为零 → 所有样本都变成支持向量 → 过拟合/躺平。
|
||||
# TransformedTargetRegressor 在 fit 时自动 StandardScaler(y),
|
||||
# predict 时自动 inverse_transform,对调用方完全透明。
|
||||
_is_svr = (config['model'] == SVR)
|
||||
if _is_svr:
|
||||
base_model = TransformedTargetRegressor(
|
||||
regressor=base_model,
|
||||
transformer=StandardScaler()
|
||||
)
|
||||
|
||||
# ============ 关键:把预处理器塞进 Pipeline ============
|
||||
# DualStream_MNF / Physical_Only 需要波长列表,供 PhysicalFeatureExtractor 定位波段
|
||||
_wl_list = None
|
||||
@ -677,9 +690,16 @@ class WaterQualityModelingBatch:
|
||||
|
||||
# 以「步骤名__参数名」的格式索引参数网格;
|
||||
# config['params'] 是模型层的(无 __),统一加 model__ 前缀。
|
||||
prefixed_params = {
|
||||
f"model__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
# ★ SVR 被 TransformedTargetRegressor 包裹后,实际模型在 model.regressor_ 下,
|
||||
# GridSearchCV 参数路径变为 model__regressor__{param}
|
||||
if _is_svr:
|
||||
prefixed_params = {
|
||||
f"model__regressor__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
else:
|
||||
prefixed_params = {
|
||||
f"model__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
|
||||
# 全量网格搜索:SVR 超参组合仅 216 种(4×6×3×3),
|
||||
# 穷举远优于 RandomizedSearchCV(n_iter=10) 的随机抽样
|
||||
@ -921,8 +941,16 @@ class WaterQualityModelingBatch:
|
||||
@staticmethod
|
||||
def _find_svr_in_pipeline(pipeline):
|
||||
model_step = pipeline.named_steps.get('model')
|
||||
if model_step is not None and model_step.__class__.__name__ == 'SVR':
|
||||
if model_step is None:
|
||||
return None
|
||||
# 直接就是 SVR(旧模型 / 非 SVR 的其他回归器)
|
||||
if model_step.__class__.__name__ == 'SVR':
|
||||
return model_step
|
||||
# ★ TransformedTargetRegressor 包裹的 SVR(修复 1 引入)
|
||||
if hasattr(model_step, 'regressor_'):
|
||||
inner = model_step.regressor_
|
||||
if inner.__class__.__name__ == 'SVR':
|
||||
return inner
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -2,6 +2,7 @@ import numpy as np
|
||||
import pandas as pd
|
||||
import joblib
|
||||
import os
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Union, Tuple, Optional
|
||||
import warnings
|
||||
@ -546,12 +547,18 @@ class WaterQualityInference:
|
||||
if actual_preprocess_method.lower() in ['nan', 'none', '']:
|
||||
actual_preprocess_method = "None"
|
||||
|
||||
print(f"正在应用预处理方法: {actual_preprocess_method}")
|
||||
print(f"原始光谱数据形状: {spectra.shape}")
|
||||
|
||||
model = self.loaded_model_data['model']
|
||||
metadata = self.loaded_model_data.get('metadata', {})
|
||||
|
||||
# ★ 根据模型类型确定预处理归属
|
||||
if isinstance(model, Pipeline):
|
||||
print(f"[模型信息] 训练预处理方法: {actual_preprocess_method}"
|
||||
f" — 由 Pipeline 内部自动执行,推理端仅做光谱重采样+列对齐")
|
||||
else:
|
||||
print(f"[模型信息] 训练预处理方法: {actual_preprocess_method}"
|
||||
f" — 旧裸模型,推理端手动应用外部 Preprocessing")
|
||||
print(f"原始光谱数据形状: {spectra.shape}")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ★ DualStream_MNF:纯光谱重采样 → pipeline.predict() 全自动
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
@ -633,24 +640,67 @@ class WaterQualityInference:
|
||||
# ── 兼容旧模型(无 train_wavelengths)──
|
||||
print("[光谱重采样] 模型无 train_wavelengths 元数据(旧模型),"
|
||||
"跳过光谱重采样,进入兼容路径...")
|
||||
# 旧兼容逻辑:WQI 自动补全
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
|
||||
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
formulas = calc.list_available()
|
||||
if formulas:
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
|
||||
f"(+{spectra.shape[1] - original_col_count} WQI)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败: {e}")
|
||||
# 旧兼容逻辑:WQI 自动补全(仅对旧裸模型生效;Pipeline 走 train_columns 对齐)
|
||||
if isinstance(model, Pipeline):
|
||||
train_cols = metadata.get('train_columns', None)
|
||||
if train_cols is not None and len(train_cols) > 0:
|
||||
print(f"[兼容+Pipeline 列对齐] 训练列数: {len(train_cols)}, 推理当前列数: {spectra.shape[1]}")
|
||||
spectra_cols = [str(c) for c in spectra.columns]
|
||||
mapping = {}
|
||||
unmatched = []
|
||||
for tc in train_cols:
|
||||
tc_str = str(tc)
|
||||
if tc_str in spectra_cols:
|
||||
mapping[tc_str] = tc_str
|
||||
else:
|
||||
unmatched.append(tc_str)
|
||||
still_unmatched = []
|
||||
for tc_str in unmatched:
|
||||
tc_float = None
|
||||
try: tc_float = float(tc_str)
|
||||
except (ValueError, TypeError): pass
|
||||
if tc_float is not None:
|
||||
found = None
|
||||
for sc in spectra_cols:
|
||||
if sc in mapping: continue
|
||||
try:
|
||||
if math.isclose(tc_float, float(sc), rel_tol=1e-4):
|
||||
found = sc; break
|
||||
except (ValueError, TypeError): pass
|
||||
if found is not None:
|
||||
mapping[found] = tc_str; continue
|
||||
still_unmatched.append(tc_str)
|
||||
aligned_parts = []
|
||||
for sc in spectra_cols:
|
||||
if sc in mapping:
|
||||
col_data = spectra[sc].copy()
|
||||
col_data.name = mapping[sc]
|
||||
aligned_parts.append(col_data)
|
||||
spectra_aligned = pd.concat(aligned_parts, axis=1) if aligned_parts else pd.DataFrame(index=spectra.index)
|
||||
for tc_str in still_unmatched:
|
||||
spectra_aligned[tc_str] = 0.0
|
||||
spectra = spectra_aligned[[str(c) for c in train_cols]]
|
||||
print(f"[兼容+Pipeline 列对齐] 完成 → {spectra.shape[1]} 列")
|
||||
else:
|
||||
print("[兼容+Pipeline] 无 train_columns,跳过 WQI 补全")
|
||||
else:
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
|
||||
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
formulas = calc.list_available()
|
||||
if formulas:
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
|
||||
f"(+{spectra.shape[1] - original_col_count} WQI)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败: {e}")
|
||||
|
||||
# ★ v4: 无 train_wavelengths 时,基于物理波长的经验重采样
|
||||
# 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格
|
||||
@ -720,26 +770,97 @@ class WaterQualityInference:
|
||||
f"{n_current} → {expected_features} 列")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ★ 特征补全:模型训练时可能包含 WQI 指数等衍生特征,
|
||||
# 推理端需自动计算补齐(适用于新旧模型两条路径)
|
||||
# ★ 特征列精确对齐:训练端可能包含 WQI 指数等衍生特征,
|
||||
# 推理端必须精确匹配训练时的列集合,不能盲目补全。
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
|
||||
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
formulas = calc.list_available()
|
||||
if formulas:
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
|
||||
f"(+{spectra.shape[1] - original_col_count} WQI)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败: {e}")
|
||||
if isinstance(model, Pipeline):
|
||||
# ── Pipeline 模型:用 train_columns 做精确列对齐 ──
|
||||
train_cols = metadata.get('train_columns', None)
|
||||
if train_cols is not None and len(train_cols) > 0:
|
||||
print(f"[Pipeline 列对齐] 训练列数: {len(train_cols)}, 推理当前列数: {spectra.shape[1]}")
|
||||
# 构建 推理列名 → 训练列名 的映射
|
||||
spectra_cols = [str(c) for c in spectra.columns]
|
||||
mapping: Dict[str, str] = {} # spectra_col_name → train_col_name
|
||||
train_set = set(str(tc) for tc in train_cols)
|
||||
|
||||
# 第一遍:精确字符串匹配
|
||||
unmatched_train = []
|
||||
for tc in train_cols:
|
||||
tc_str = str(tc)
|
||||
if tc_str in spectra_cols:
|
||||
mapping[tc_str] = tc_str
|
||||
else:
|
||||
unmatched_train.append(tc_str)
|
||||
|
||||
# 第二遍:对未匹配的训练列,尝试浮点数近似匹配
|
||||
still_unmatched = []
|
||||
for tc_str in unmatched_train:
|
||||
tc_float = None
|
||||
try:
|
||||
tc_float = float(tc_str)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if tc_float is not None:
|
||||
found = None
|
||||
for sc in spectra_cols:
|
||||
if sc in mapping: # 已经被匹配过了
|
||||
continue
|
||||
try:
|
||||
sc_float = float(sc)
|
||||
if math.isclose(tc_float, sc_float, rel_tol=1e-4):
|
||||
found = sc
|
||||
break
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
if found is not None:
|
||||
mapping[found] = tc_str
|
||||
continue
|
||||
still_unmatched.append(tc_str)
|
||||
|
||||
# 从 spectra 中挑选已匹配的列,并重命名为训练列名
|
||||
aligned_parts = []
|
||||
matched_train = set(mapping.values())
|
||||
for sc in spectra_cols:
|
||||
if sc in mapping:
|
||||
col_data = spectra[sc].copy()
|
||||
col_data.name = mapping[sc] # 重命名为训练列名
|
||||
aligned_parts.append(col_data)
|
||||
|
||||
spectra_aligned = pd.concat(aligned_parts, axis=1) if aligned_parts else pd.DataFrame(index=spectra.index)
|
||||
|
||||
# 对仍未匹配的训练列:WQI_ 补零,其他警告
|
||||
for tc_str in still_unmatched:
|
||||
spectra_aligned[tc_str] = 0.0
|
||||
|
||||
# 最终按 train_cols 顺序排列输出
|
||||
spectra = spectra_aligned[[str(c) for c in train_cols]]
|
||||
matched_count = len(matched_train)
|
||||
missing_count = len(still_unmatched)
|
||||
print(f"[Pipeline 列对齐] 精确匹配 {matched_count - missing_count} 列, "
|
||||
f"浮点近似匹配 {len(mapping) - (matched_count - missing_count)} 列, "
|
||||
f"补零填充 {missing_count} 列 → 最终 {spectra.shape[1]} 列")
|
||||
else:
|
||||
print("[Pipeline 列对齐] 无 train_columns 元数据,跳过列对齐")
|
||||
else:
|
||||
# ── 旧裸模型:保留原有 WQI 自动补全逻辑 ──
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
|
||||
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
formulas = calc.list_available()
|
||||
if formulas:
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
|
||||
f"(+{spectra.shape[1] - original_col_count} WQI)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败: {e}")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 通用清洗
|
||||
@ -1009,6 +1130,33 @@ class WaterQualityInference:
|
||||
print("-" * 40)
|
||||
coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path)
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# ★ 自适应反射率量级缩放 (Scale Alignment)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# 不同的高光谱传感器 / 处理流程产出的反射率量级可能不同:
|
||||
# - float32 0-1 物理反射率(如 result3.bsq 抽样后写入的 CSV)
|
||||
# - int16 0-10000 放大反射率(如 ref_mosaic 抽样后写入的 CSV)
|
||||
# 若不经缩放直接喂入 SVR,量级差异会导致预测完全失效。
|
||||
# 此处在光谱列上自动检测并统一到 0-1 区间。
|
||||
spec_cols = []
|
||||
for c in spectra.columns:
|
||||
try:
|
||||
float(str(c))
|
||||
spec_cols.append(c)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if spec_cols:
|
||||
max_val = spectra[spec_cols].max().max()
|
||||
if max_val > 10:
|
||||
print(f"\n[量级检测] 输入反射率疑似放大格式 (max={max_val:.2f})")
|
||||
print("[量级检测] 自动除以 10000,缩放至 0-1 标准物理反射率区间...")
|
||||
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
|
||||
print(f"[量级检测] 缩放完成!缩放后 max={spectra[spec_cols].max().max():.4f}")
|
||||
else:
|
||||
print(f"[量级检测] 输入反射率量级正常 (max={max_val:.4f}),无需缩放")
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
# 3. 数据预处理
|
||||
print("\n步骤3: 数据预处理")
|
||||
print("-" * 40)
|
||||
@ -1097,12 +1245,21 @@ class WaterQualityInference:
|
||||
|
||||
# 执行推理
|
||||
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
|
||||
# 自适应反射率量级缩放
|
||||
_s_cols = []
|
||||
for _c in spectra.columns:
|
||||
try: float(str(_c)); _s_cols.append(_c)
|
||||
except (ValueError, TypeError): pass
|
||||
if _s_cols:
|
||||
_mv = spectra[_s_cols].max().max()
|
||||
if _mv > 10:
|
||||
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
|
||||
spectra_processed = self.preprocess_spectra(spectra)
|
||||
predictions = self.predict(spectra_processed)
|
||||
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
|
||||
result_df = self.save_predictions(coords, predictions, str(output_file),
|
||||
prediction_column, wqi_df)
|
||||
|
||||
|
||||
results[csv_file.name] = {
|
||||
'output_file': str(output_file),
|
||||
'sample_count': len(predictions),
|
||||
@ -1297,12 +1454,21 @@ class WaterQualityInference:
|
||||
|
||||
# 执行推理
|
||||
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
|
||||
# 自适应反射率量级缩放
|
||||
_s_cols = []
|
||||
for _c in spectra.columns:
|
||||
try: float(str(_c)); _s_cols.append(_c)
|
||||
except (ValueError, TypeError): pass
|
||||
if _s_cols:
|
||||
_mv = spectra[_s_cols].max().max()
|
||||
if _mv > 10:
|
||||
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
|
||||
spectra_processed = self.preprocess_spectra(spectra)
|
||||
predictions = self.predict(spectra_processed)
|
||||
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
|
||||
result_df = self.save_predictions(coords, predictions, str(output_file),
|
||||
prediction_column, wqi_df)
|
||||
|
||||
|
||||
results[file_stem] = {
|
||||
'input_file': str(csv_file),
|
||||
'output_file': str(output_file),
|
||||
|
||||
@ -233,6 +233,14 @@ class Step9MlPredictPanel(QWidget):
|
||||
return
|
||||
|
||||
self.external_model_dir = dir_path
|
||||
|
||||
# ★ 读取用户选择的模型评估指标
|
||||
_metric_key = self.metric.currentData() # "test_r2" / "test_rmse" / "test_mae"
|
||||
_metric_display = self.metric.currentText() # "R² (决定系数)" / ...
|
||||
_is_r2 = 'r2' in _metric_key.lower()
|
||||
_is_rmse = 'rmse' in _metric_key.lower()
|
||||
_is_mae = 'mae' in _metric_key.lower()
|
||||
|
||||
models_found = {}
|
||||
errors = []
|
||||
|
||||
@ -243,26 +251,53 @@ class Step9MlPredictPanel(QWidget):
|
||||
if not subentry.is_dir():
|
||||
continue
|
||||
subdir_name = subentry.name
|
||||
joblib_files = [
|
||||
joblib_files = sorted([
|
||||
f for f in os.scandir(subentry.path)
|
||||
if f.is_file() and f.name.lower().endswith(".joblib")
|
||||
]
|
||||
], key=lambda x: x.name)
|
||||
if not joblib_files:
|
||||
continue
|
||||
joblib_path = joblib_files[0].path
|
||||
try:
|
||||
loaded = joblib.load(joblib_path)
|
||||
if isinstance(loaded, dict) and "model" in loaded:
|
||||
# ★ 保留完整 dict(含 metadata / train_wavelengths),
|
||||
# 推理端需要 train_wavelengths 做光谱重采样
|
||||
models_found[subdir_name] = loaded
|
||||
elif hasattr(loaded, "predict"):
|
||||
models_found[subdir_name] = loaded
|
||||
else:
|
||||
errors.append(f"{subdir_name}: 无法识别的格式 {type(loaded).__name__}")
|
||||
|
||||
# ★ 遍历该子目录下全部 .joblib 文件,按 metric 选出最佳模型
|
||||
best_score = -float('inf') if _is_r2 else float('inf')
|
||||
best_entry = None
|
||||
best_fname = None
|
||||
|
||||
for f_entry in joblib_files:
|
||||
try:
|
||||
data = joblib.load(f_entry.path)
|
||||
if not isinstance(data, dict) or "model" not in data:
|
||||
continue
|
||||
meta = data.get('metadata', {})
|
||||
# 按指标优先级:test_xxx → train_xxx
|
||||
score = None
|
||||
if _is_r2:
|
||||
score = meta.get('test_r2', meta.get('train_r2', None))
|
||||
elif _is_rmse:
|
||||
score = meta.get('test_rmse', meta.get('train_rmse', None))
|
||||
elif _is_mae:
|
||||
score = meta.get('test_mae', meta.get('train_mae', None))
|
||||
|
||||
if score is None:
|
||||
continue
|
||||
|
||||
if _is_r2 and score > best_score:
|
||||
best_score = score
|
||||
best_entry = data
|
||||
best_fname = f_entry.name
|
||||
elif not _is_r2 and score < best_score:
|
||||
best_score = score
|
||||
best_entry = data
|
||||
best_fname = f_entry.name
|
||||
except Exception:
|
||||
continue
|
||||
except Exception as e:
|
||||
errors.append(f"{subdir_name}: {type(e).__name__}: {e}")
|
||||
|
||||
if best_entry is not None:
|
||||
models_found[subdir_name] = best_entry
|
||||
print(f"[模型加载] 目标 {subdir_name} 根据 {_metric_display} "
|
||||
f"自动选择最佳模型: {best_fname}, 分数: {best_score:.6f}")
|
||||
else:
|
||||
errors.append(f"{subdir_name}: 无可评估的 .joblib 文件")
|
||||
|
||||
except Exception as e:
|
||||
QMessageBox.warning(
|
||||
|
||||
@ -381,8 +381,28 @@ class MNFTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
n_samples, n_features = X.shape
|
||||
|
||||
# 1) 噪声矩阵:相邻行差分
|
||||
noise = X[1:] - X[:-1]
|
||||
# 1) ★ 致命修复 3:噪声矩阵估计。
|
||||
# 旧方案 noise = X[1:] - X[:-1](相邻行差分)假设数据是空间连续的图像,
|
||||
# 相邻像素差异 = 传感器噪声。但进入 Pipeline.fit() 的 X_train 已被
|
||||
# split_data() 随机抽取 + KFold(shuffle=True) 打乱,相邻行可能是空间上
|
||||
# 相隔十万八千里的不同水质样本。此时差分结果是自然方差而非噪声,
|
||||
# 白化操作会把真实水质信号当噪声除掉。
|
||||
# 新方案:Savitzky-Golay 滤波提取光谱维平滑曲线,残差 = 原始 - 平滑 ≈ 纯传感器噪声。
|
||||
from scipy.signal import savgol_filter
|
||||
try:
|
||||
# window_length 必须是奇数,取 7(或波段数的一半向下取奇数)
|
||||
_wl = min(7, n_features - 1)
|
||||
if _wl % 2 == 0:
|
||||
_wl -= 1
|
||||
if _wl >= 5:
|
||||
X_smooth = savgol_filter(X, window_length=_wl, polyorder=2, axis=1)
|
||||
noise = X - X_smooth
|
||||
else:
|
||||
# 波段太少,无法滤波,回退到去均值残差
|
||||
noise = X - np.mean(X, axis=1, keepdims=True)
|
||||
except Exception:
|
||||
# 兜底:去均值残差
|
||||
noise = X - np.mean(X, axis=1, keepdims=True)
|
||||
Cn = np.cov(noise, rowvar=False)
|
||||
_reg = np.trace(Cn) / n_features * 0.01
|
||||
Cn += np.eye(n_features) * max(_reg, 1e-6)
|
||||
@ -611,10 +631,18 @@ def get_preprocessing_transformer(method: str, wavelengths=None,
|
||||
return IdentityTransformer()
|
||||
if method == "DualStream_MNF":
|
||||
from sklearn.pipeline import FeatureUnion
|
||||
return FeatureUnion([
|
||||
from sklearn.pipeline import Pipeline as _Pipeline
|
||||
# ★ 致命修复 2:MNF 主成分(量级 ~±10)与 Physical 指数(量级 ~0.001)
|
||||
# 尺度差异可达 10^4 倍,SVR 的 RBF 距离会被大值特征主导。
|
||||
# FeatureUnion 之后必须接 StandardScaler 统一量纲。
|
||||
mnf_union = FeatureUnion([
|
||||
('physical', PhysicalFeatureExtractor(wavelengths=wavelengths)),
|
||||
('mnf', MNFTransformer(n_components=mnf_n_components)),
|
||||
])
|
||||
return _Pipeline([
|
||||
('features', mnf_union),
|
||||
('scaler', StandardScaler()),
|
||||
])
|
||||
if method == "Physical_Only":
|
||||
from sklearn.pipeline import Pipeline as _Pipeline
|
||||
return _Pipeline([
|
||||
|
||||
Reference in New Issue
Block a user