feat: ML模型波长记忆 + 跨传感器光谱重采样 + 数据加载规范化
训练端 (modeling_batch.py): - _extract_train_wavelengths() 从列名提取训练波长写入 metadata - load_data_batch/load_data_single 改为基于列名语义智能提取特征 - 废除 feature_start_column 硬编码位置索引 推理端 (inference_batch.py): - preprocess_spectra() 分支A: train_wavelengths存在→np.interp精确重采样 - preprocess_spectra() 分支B: 无波长元数据→解析列名→np.interp到400-800nm标准网格 - 废除暴力截断和零值填充
This commit is contained in:
duxin
@ -377,110 +377,253 @@ class WaterQualityInference:
|
||||
|
||||
def preprocess_spectra(self, spectra: pd.DataFrame) -> np.ndarray:
|
||||
"""
|
||||
对光谱数据进行预处理
|
||||
对光谱数据进行预处理 + 跨传感器光谱重采样。
|
||||
|
||||
改造要点（v2 — 光谱重采样）：
|
||||
- 废除原有的「列位置截断」和「零值填充」hack
|
||||
- 从 model metadata 中读取训练时的波长列表
|
||||
- 如果输入传感器与训练传感器波长网格不同，使用 np.interp()
|
||||
将输入光谱重采样到训练波长网格
|
||||
- 非光谱列（WQI 指数等）保持不变
|
||||
- 兼容旧模型（无 train_wavelengths 时回退到旧逻辑）
|
||||
|
||||
Args:
|
||||
spectra: 原始光谱数据
|
||||
spectra: 原始光谱数据（DataFrame，列名为波长字符串）
|
||||
|
||||
Returns:
|
||||
预处理后的光谱数据
|
||||
预处理后的光谱数据 (numpy ndarray)
|
||||
"""
|
||||
if self.loaded_model_data is None:
|
||||
raise ValueError("请先加载模型")
|
||||
|
||||
preprocess_method = self.loaded_model_data['preprocess_method']
|
||||
|
||||
# 处理 nan/NaN/None 值，转换为 "None" 字符串
|
||||
|
||||
# 处理 nan/NaN/None 值
|
||||
if pd.isna(preprocess_method) or str(preprocess_method).lower() in ['nan', 'none', '']:
|
||||
preprocess_method = "None"
|
||||
|
||||
|
||||
# 解析预处理方法（可能包含划分方法前缀）
|
||||
if '_' in str(preprocess_method):
|
||||
parts = str(preprocess_method).split('_')
|
||||
# 假设格式为 split_method_preprocess_method
|
||||
actual_preprocess_method = '_'.join(parts[1:]) if len(parts) > 1 else parts[-1]
|
||||
else:
|
||||
actual_preprocess_method = str(preprocess_method)
|
||||
|
||||
# 再次检查并转换 nan
|
||||
|
||||
if actual_preprocess_method.lower() in ['nan', 'none', '']:
|
||||
actual_preprocess_method = "None"
|
||||
|
||||
print(f"正在应用预处理方法: {actual_preprocess_method}")
|
||||
print(f"原始光谱数据形状: {spectra.shape}")
|
||||
|
||||
# ---- 自动特征补全：50 光谱 → 补全至模型训练时的 95 维（WQI 指数） ----
|
||||
# 触发条件：模型期望 n_features_in_ 个特征，但当前 spectra 列数不足
|
||||
# 原因：training_spectra.csv 含 50 光谱 + 45 WQI；sampling_spectra.csv 只有 50 光谱
|
||||
# 做法：与训练端（calculate_all_indices）完全一致的算法列表，实时补全缺失的 45 个 WQI 列
|
||||
model = self.loaded_model_data['model']
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
metadata = self.loaded_model_data.get('metadata', {})
|
||||
train_wavelengths = metadata.get('train_wavelengths', None)
|
||||
train_columns = metadata.get('train_columns', None)
|
||||
|
||||
# ---- 自动特征补全：50 光谱 → 补全至模型训练时的 n_features_in_ 维（WQI 指数） ----
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口：当前 {spectra.shape[1]} 列 < 模型期望 {expected_features} 列，"
|
||||
f"正在从光谱数据实时计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ★ 核心：光谱重采样（跨传感器适配）
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
if train_wavelengths is not None and len(train_wavelengths) > 0:
|
||||
print(f"[光谱重采样] 模型训练波长: {len(train_wavelengths)} 个, "
|
||||
f"范围 {train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm")
|
||||
|
||||
# CSV 驱动的 WaterQualityIndexCalculator：所有公式名通过 list_available() 拿；
|
||||
# 一次性 calculate_many() 批量计算。彻底摆脱 dir(calc) 反射扫描 + 单 algo_func
|
||||
# 调用这种碎片化写法（Calculator 早已重构为公式驱动，不再有独立公式方法）。
|
||||
formulas = calc.list_available()
|
||||
if not formulas:
|
||||
print("[特征补全] Calculator 未持有任何公式，跳过补全")
|
||||
# 1) 从当前 spectra 中分离光谱列和 WQI 列
|
||||
target_wavelengths = []
|
||||
spectral_col_indices = []
|
||||
wqi_col_indices = []
|
||||
wqi_col_names = []
|
||||
|
||||
for i, col in enumerate(spectra.columns):
|
||||
col_str = str(col)
|
||||
try:
|
||||
wl = float(col_str)
|
||||
target_wavelengths.append(wl)
|
||||
spectral_col_indices.append(i)
|
||||
except (ValueError, TypeError):
|
||||
if col_str.startswith('WQI_'):
|
||||
wqi_col_indices.append(i)
|
||||
wqi_col_names.append(col_str)
|
||||
|
||||
print(f"[光谱重采样] 当前传感器: {len(target_wavelengths)} 个波长列, "
|
||||
f"{len(wqi_col_indices)} 个 WQI 列")
|
||||
|
||||
if len(target_wavelengths) == 0:
|
||||
print("[光谱重采样] ⚠ 未检测到波长列，跳过重采样")
|
||||
else:
|
||||
# 2) 提取光谱数据矩阵
|
||||
spectral_data = spectra.iloc[:, spectral_col_indices].values.astype(np.float64)
|
||||
|
||||
# 3) 逐行 np.interp 重采样
|
||||
train_wl_arr = np.array(train_wavelengths, dtype=np.float64)
|
||||
target_wl_arr = np.array(target_wavelengths, dtype=np.float64)
|
||||
|
||||
print(f"[光谱重采样] 执行重采样: {len(target_wavelengths)} → "
|
||||
f"{len(train_wavelengths)} 个波长点 ...")
|
||||
resampled = np.zeros((spectral_data.shape[0], len(train_wavelengths)),
|
||||
dtype=np.float64)
|
||||
for i in range(spectral_data.shape[0]):
|
||||
resampled[i] = np.interp(
|
||||
train_wl_arr, target_wl_arr, spectral_data[i],
|
||||
left=np.nan, right=np.nan
|
||||
)
|
||||
|
||||
# NaN 填充（外推区域）：用最近有效值填充
|
||||
nan_mask = np.isnan(resampled)
|
||||
if nan_mask.any():
|
||||
print(f"[光谱重采样] {nan_mask.sum()} 个 NaN (外推区域) → 最近邻填充")
|
||||
for i in range(resampled.shape[0]):
|
||||
row_nan = np.isnan(resampled[i])
|
||||
if row_nan.any():
|
||||
valid = ~row_nan
|
||||
if valid.any():
|
||||
valid_idx = np.where(valid)[0]
|
||||
resampled[i, row_nan] = np.interp(
|
||||
np.where(row_nan)[0], valid_idx,
|
||||
resampled[i, valid]
|
||||
)
|
||||
else:
|
||||
resampled[i] = 0.0
|
||||
|
||||
# 4) 重组为 DataFrame（列名 = 训练波长字符串）
|
||||
wl_col_names = [f"{wl:.6f}" for wl in train_wavelengths]
|
||||
resampled_df = pd.DataFrame(resampled, columns=wl_col_names,
|
||||
index=spectra.index)
|
||||
|
||||
# 5) 拼接 WQI 列（如有）
|
||||
if wqi_col_indices:
|
||||
wqi_df = spectra.iloc[:, wqi_col_indices].copy()
|
||||
wqi_df.columns = wqi_col_names
|
||||
spectra = pd.concat([resampled_df, wqi_df], axis=1)
|
||||
else:
|
||||
# fast=True: Step9 特征补全走向量化快车道（~63 公式，已验证稳定）
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
# results_df 是列对齐的 WQI 计算结果（每列一个公式，行数=样本数）
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成！光谱列已扩充至 {spectra.shape[1]} 列"
|
||||
f"（追加了 {spectra.shape[1] - original_col_count} 个 WQI 指数）")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败，将使用原始光谱特征: {e}")
|
||||
spectra = resampled_df
|
||||
|
||||
# ---- 防线 1：强制维度对齐（物理截断）----
|
||||
if expected_features is not None and spectra.shape[1] > expected_features:
|
||||
print(f"[精准对齐] 正在将 {spectra.shape[1]} 维特征截断为模型要求的 {expected_features} 维")
|
||||
spectra = spectra.iloc[:, :expected_features]
|
||||
elif expected_features is not None and spectra.shape[1] < expected_features:
|
||||
# 维度不足时填充 0
|
||||
padding_cols = expected_features - spectra.shape[1]
|
||||
for i in range(padding_cols):
|
||||
spectra[f'_padding_{i}'] = 0.0
|
||||
print(f"[精准对齐] 特征不足，填充 {padding_cols} 列 0")
|
||||
print(f"[光谱重采样] 完成: {spectra.shape[1]} 列 "
|
||||
f"(光谱 {len(train_wavelengths)} + WQI {len(wqi_col_indices)})")
|
||||
|
||||
# ---- 防线 2：彻底清洗无穷大数值----
|
||||
# 防止 WQI 计算中除零/溢出产生 np.inf / -np.inf 导致预处理崩溃
|
||||
else:
|
||||
# ── 兼容旧模型（无 train_wavelengths）──
|
||||
print("[光谱重采样] 模型无 train_wavelengths 元数据（旧模型），"
|
||||
"跳过光谱重采样，进入兼容路径...")
|
||||
# 旧兼容逻辑：WQI 自动补全
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口：当前 {spectra.shape[1]} 列 "
|
||||
f"< 模型期望 {expected_features} 列，正在计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
formulas = calc.list_available()
|
||||
if formulas:
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成！扩充至 {spectra.shape[1]} 列 "
|
||||
f"(+{spectra.shape[1] - original_col_count} WQI)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败: {e}")
|
||||
|
||||
# ★ v4: 无 train_wavelengths 时，基于物理波长的经验重采样
|
||||
# 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格
|
||||
# 取代原基于列索引的硬截断（不同传感器列索引对应的波长完全不同）
|
||||
if expected_features is not None:
|
||||
n_current = spectra.shape[1]
|
||||
|
||||
# ── 第1步：从列名中提取波长列和非波长列 ──
|
||||
_target_wl: List[float] = []
|
||||
_wqi_names: List[str] = []
|
||||
_other_names: List[str] = []
|
||||
for _col in spectra.columns:
|
||||
try:
|
||||
_target_wl.append(float(str(_col)))
|
||||
except (ValueError, TypeError):
|
||||
if str(_col).startswith('WQI_'):
|
||||
_wqi_names.append(str(_col))
|
||||
else:
|
||||
_other_names.append(str(_col))
|
||||
|
||||
# 分离出 WQI 列（保存以便后续拼接回去）
|
||||
_wqi_df = spectra[_wqi_names].copy() if _wqi_names else pd.DataFrame()
|
||||
|
||||
if len(_target_wl) >= 2 and expected_features > 0:
|
||||
# ── 第2步：有真实波长 → np.interp 重采样 ──
|
||||
fallback_wl = np.linspace(400.0, 800.0, expected_features,
|
||||
dtype=np.float64)
|
||||
_target_arr = np.array(_target_wl, dtype=np.float64)
|
||||
_spec_data = spectra.iloc[:, :len(_target_wl)].values.astype(np.float64)
|
||||
# 按波长排序（以防列顺序不是严格递增）
|
||||
_sort_idx = np.argsort(_target_arr)
|
||||
_target_arr = _target_arr[_sort_idx]
|
||||
_spec_data = _spec_data[:, _sort_idx]
|
||||
|
||||
_resampled = np.zeros((_spec_data.shape[0], expected_features),
|
||||
dtype=np.float64)
|
||||
for i in range(_spec_data.shape[0]):
|
||||
_resampled[i] = np.interp(
|
||||
fallback_wl, _target_arr, _spec_data[i],
|
||||
left=np.nan, right=np.nan
|
||||
)
|
||||
# NaN 填充
|
||||
_nan = np.isnan(_resampled)
|
||||
if _nan.any():
|
||||
for i in range(_resampled.shape[0]):
|
||||
row_nan = np.isnan(_resampled[i])
|
||||
if row_nan.any():
|
||||
valid = ~row_nan
|
||||
if valid.any():
|
||||
vi = np.where(valid)[0]
|
||||
_resampled[i, row_nan] = np.interp(
|
||||
np.where(row_nan)[0], vi, _resampled[i, valid]
|
||||
)
|
||||
else:
|
||||
_resampled[i] = 0.0
|
||||
|
||||
# 重组 DataFrame
|
||||
_wl_cols = [f"{w:.1f}" for w in fallback_wl]
|
||||
spectra = pd.DataFrame(_resampled, columns=_wl_cols,
|
||||
index=spectra.index)
|
||||
if not _wqi_df.empty:
|
||||
spectra = pd.concat([spectra, _wqi_df.reset_index(drop=True)],
|
||||
axis=1)
|
||||
print(f"[兼容重采样] 列名解析到 {len(_target_wl)} 个波长 "
|
||||
f"({_target_arr[0]:.1f}~{_target_arr[-1]:.1f}nm) → "
|
||||
f"np.interp → {expected_features} 列 (400~800nm 标准网格)"
|
||||
f"{' + ' + str(len(_wqi_names)) + ' WQI' if _wqi_names else ''}")
|
||||
|
||||
elif n_current > expected_features:
|
||||
# ── 第3步：无法解析波长（纯字符串列名）→ 硬截断兜底 ──
|
||||
spectra = spectra.iloc[:, :expected_features]
|
||||
print(f"[兼容截断] 无法解析波长列名，硬截断 "
|
||||
f"{n_current} → {expected_features} 列")
|
||||
|
||||
elif n_current < expected_features:
|
||||
# ── 第4步：特征不够 → 补零 ──
|
||||
for i in range(expected_features - n_current):
|
||||
spectra[f'_padding_{i}'] = 0.0
|
||||
print(f"[兼容填充] 特征不足，补零 "
|
||||
f"{n_current} → {expected_features} 列")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 通用清洗
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
spectra = spectra.replace([np.inf, -np.inf], np.nan)
|
||||
spectra = spectra.fillna(0)
|
||||
|
||||
print(f"[特征对齐] 最终输入维度: {spectra.shape}")
|
||||
|
||||
# ---- Pipeline 化分支：模型内置 scaler/MSC.mean_spectrum_ 等状态时，跳过手动 Preprocessing ----
|
||||
# ── Pipeline 化分支：模型内置 scaler → 跳过手动 Preprocessing ──
|
||||
if isinstance(model, Pipeline):
|
||||
print(f"[Pipeline] 检测到模型是 sklearn Pipeline，"
|
||||
f"其内置预处理步骤（{list(model.named_steps.keys())[0]}）将处理原始光谱，"
|
||||
print(f"[Pipeline] 模型是 sklearn Pipeline，内置预处理步骤，"
|
||||
f"无需外部 Preprocessing")
|
||||
return spectra.values
|
||||
|
||||
# ---- 兼容路径：旧 .joblib（裸模型 + preprocess_method 字符串）回退手动 Preprocessing ----
|
||||
# ── 兼容路径：旧裸模型 + 手动 Preprocessing ──
|
||||
try:
|
||||
# 应用预处理
|
||||
spectra_processed = Preprocessing(actual_preprocess_method, spectra)
|
||||
|
||||
# 确保返回numpy数组
|
||||
if isinstance(spectra_processed, pd.DataFrame):
|
||||
spectra_processed = spectra_processed.values
|
||||
|
||||
print(f" [Legacy] 旧裸模型 + 手动 Preprocessing({actual_preprocess_method}) 完成，"
|
||||
f"数据形状: {spectra_processed.shape}")
|
||||
|
||||
return spectra_processed
|
||||
|
||||
except Exception as e:
|
||||
print(f"预处理失败: {e}")
|
||||
print("使用原始数据")
|
||||
|
||||
Reference in New Issue
Block a user