feat: ML模型波长记忆 + 跨传感器光谱重采样 + 数据加载规范化

训练端 (modeling_batch.py):
- _extract_train_wavelengths() 从列名提取训练波长写入 metadata
- load_data_batch/load_data_single 改为基于列名语义智能提取特征
- 废除 feature_start_column 硬编码位置索引

推理端 (inference_batch.py):
- preprocess_spectra() 分支A: train_wavelengths存在→np.interp精确重采样
- preprocess_spectra() 分支B: 无波长元数据→解析列名→np.interp到400-800nm标准网格
- 废除暴力截断和零值填充
This commit is contained in:
duxin
2026-07-27 14:52:30 +08:00
parent 99aeab3076
commit 92e8c90370
7 changed files with 398 additions and 149 deletions

View File

@ -32,7 +32,7 @@ class Step8MlTrainHandler(BaseStepHandler):
try:
result = ModelingStep.train_models(
feature_start_column=config.get('feature_start_column', '374.285004'),
feature_start_column=config.get('feature_start_column', None),
preprocessing_methods=config.get('preprocessing_methods'),
model_names=config.get('model_names'),
split_methods=config.get('split_methods'),

View File

@ -231,19 +231,125 @@ class WaterQualityModelingBatch:
self.results = {}
self.best_models = {}
def load_data_batch(self, csv_path: str, feature_start_column: Union[int, str]) -> Tuple[pd.DataFrame, Dict[str, pd.Series]]:
"""
批量加载CSV数据,将指定列之前的列作为目标值
@staticmethod
def _is_wavelength_column(col_name: str) -> bool:
"""判断列名是否为波长值(纯数字字符串,如 '374.285004')"""
try:
float(str(col_name))
return True
except (ValueError, TypeError):
return False
Args:
csv_path: CSV文件路径
feature_start_column: 特征开始列索引(int)或列名(str)
@staticmethod
def _is_wqi_column(col_name: str) -> bool:
"""判断列名是否为 WQI 水质指数列('WQI_' 前缀)"""
return str(col_name).startswith('WQI_')
@staticmethod
def _extract_train_wavelengths(columns) -> List[float]:
"""从列名列表中提取波长值(float 列表)
遍历列名,将所有可转为 float 的列名提取为波长列表。
用于写入模型 metadata['train_wavelengths'],供推理端光谱重采样。
"""
wl_list = []
for c in columns:
try:
wl_list.append(float(str(c)))
except (ValueError, TypeError):
pass
return wl_list
def _extract_feature_columns(self, data: pd.DataFrame,
feature_start_column: Union[int, str, None] = None
) -> Tuple[pd.DataFrame, List[int]]:
"""从 DataFrame 中提取特征列(基于列名语义,而非位置索引)
策略(优先级从高到低):
1) 如果 feature_start_column 是列名(str),定位该列并取之后的列
2) 如果 feature_start_column 是整数,兼容旧逻辑(位置索引)
3) 如果 feature_start_column 为 None,自动识别:
- 保留所有纯数字列名(波长列)
- 保留所有 WQI_ 前缀列(水质指数列)
- 跳过坐标/元数据列
Returns:
X: 特征数据
X: 特征 DataFrame(保留列名)
feature_col_indices: 特征列在原 data 中的位置索引列表
"""
all_cols = list(data.columns)
if feature_start_column is not None:
# ── 兼容旧逻辑:按列名或索引位置截取 ──
if isinstance(feature_start_column, str):
if feature_start_column not in data.columns:
raise ValueError(
f"指定的特征开始列 '{feature_start_column}' 不存在于数据中"
)
start_idx = data.columns.get_loc(feature_start_column)
print(f"[特征提取] 按列名 '{feature_start_column}' 定位 → 索引 {start_idx}")
else:
start_idx = int(feature_start_column)
print(f"[特征提取] 按位置索引 {start_idx} 截取")
X = data.iloc[:, start_idx:]
feature_indices = list(range(start_idx, len(all_cols)))
else:
# ── 智能识别:按列名语义过滤 ──
# 黑名单:坐标列、元数据列(不会被误判为特征)
_meta_patterns = {
'x_coord', 'y_coord', 'pixel_x', 'pixel_y',
'longitude', 'latitude', 'lon', 'lat',
'id', 'station', 'sample_id',
}
feature_indices = []
for i, col in enumerate(all_cols):
col_lower = str(col).lower().strip()
# 跳过元数据列
if col_lower in _meta_patterns:
continue
# 保留波长列
if self._is_wavelength_column(col):
feature_indices.append(i)
# 保留 WQI 列
elif self._is_wqi_column(col):
feature_indices.append(i)
# 其他:跳过(可能是目标列或其他非特征列)
if not feature_indices:
raise ValueError(
"智能特征提取失败:未找到任何波长列或 WQI 列。"
f"CSV 列名: {all_cols[:10]}..."
)
X = data.iloc[:, feature_indices]
print(f"[特征提取] 智能识别: {len(feature_indices)} 个特征列 "
f"(波长列 + WQI 列)")
print(f"[特征提取] 特征数据形状: {X.shape}")
return X, feature_indices
def load_data_batch(self, csv_path: str,
feature_start_column: Union[int, str, None] = None
) -> Tuple[pd.DataFrame, Dict[str, pd.Series]]:
"""批量加载 CSV 数据,自动识别特征列与目标列
改造要点(v2):
- 特征列按列名语义提取(波长数字 / WQI_ 前缀),不再按硬编码位置一刀切
- 目标列 = 不在特征列中、且非系统保留列(ID/坐标等)的数值列
- X 保持为 DataFrame,列名保留波长信息(供后续提取 train_wavelengths)
Args:
csv_path: CSV 文件路径
feature_start_column: (可选)旧版兼容参数:
- str: 特征起始列名
- int: 特征起始列索引
- None: 自动智能识别
Returns:
X: 特征数据 (DataFrame,保留列名)
y_dict: 目标值数据字典,键为列名
"""
# 读取CSV数据,处理空字符串和缺失值
# 读取 CSV 数据,处理空字符串和缺失值
try:
data = pd.read_csv(csv_path, na_values=['', ' ', 'NaN', 'nan', 'NULL', 'null'])
except pd.errors.EmptyDataError:
@ -251,113 +357,105 @@ class WaterQualityModelingBatch:
except Exception as e:
raise ValueError(f"读取CSV文件 '{csv_path}' 时出错: {e}")
# 检查并清理数据中的空字符串和其他无效值
print("数据清理...")
original_shape = data.shape
# 将空字符串替换为NaN
# 将空字符串替换为 NaN
data = data.replace(r'^\s*$', np.nan, regex=True)
# 对于数值列,将无法转换为数字的字符串替换为NaN
# 对于数值列,将无法转换为数字的字符串替换为 NaN
for col in data.columns:
try:
# 尝试将列转换为数值类型
data[col] = pd.to_numeric(data[col], errors='coerce')
except Exception:
# 如果转换失败,保持原样(可能是字符串列)
pass
cleaned_shape = data.shape
if cleaned_shape != original_shape:
print(f"数据清理完成: {original_shape[0]}行{original_shape[1]}列 -> {cleaned_shape[0]}行{cleaned_shape[1]}列")
print(f"数据清理完成: {original_shape[0]}行{original_shape[1]}列 "
f"-> {cleaned_shape[0]}行{cleaned_shape[1]}列")
print(f"数据加载完成,总列数: {data.shape[1]}")
print(f"所有列名: {list(data.columns)}")
# 如果feature_start_column是列名,转换为索引
if isinstance(feature_start_column, str):
if feature_start_column not in data.columns:
raise ValueError(f"指定的特征开始列 '{feature_start_column}' 不存在于数据中")
feature_start_index = data.columns.get_loc(feature_start_column)
print(f"特征开始列 '{feature_start_column}' 对应索引: {feature_start_index}")
else:
feature_start_index = feature_start_column
print(f"特征开始列索引: {feature_start_index}")
# 提取特征数据(从feature_start_index开始)
X = data.iloc[:, feature_start_index:]
# 提取所有目标列(从0列到feature_start_index-1列)
print(f"所有列名: {list(data.columns)[:10]}..." if len(data.columns) > 10
else f"所有列名: {list(data.columns)}")
# ── 提取特征列(基于列名语义) ──
X, feature_indices = self._extract_feature_columns(
data, feature_start_column
)
# ── 提取目标列(不在特征列中 + 非系统保留列 + 数值类型) ──
feature_idx_set = set(feature_indices)
ignore_cols = {
'ID', 'id', 'Id',
'Longitude', 'Latitude', 'Lon', 'Lat',
'longitude', 'latitude', 'lon', 'lat',
'Station', 'station', 'sample_id',
'x_coord', 'y_coord', 'pixel_x', 'pixel_y',
}
y_dict = {}
target_columns = data.columns[:feature_start_index]
print(f"检测到的潜在目标列: {list(target_columns)}")
# 新增:跳过非预测目标的系统保留列
ignore_cols = {'ID', 'id', 'Id', 'Longitude', 'Latitude', 'Lon', 'Lat', 'longitude', 'latitude', 'lon', 'lat', 'Station', 'station'}
for col_name in target_columns:
# 过滤黑名单列
if col_name in ignore_cols:
print(f" 跳过目标列 '{col_name}': 属于系统保留列或空间坐标")
for i, col_name in enumerate(data.columns):
if i in feature_idx_set:
continue # 已在特征矩阵中
col_str = str(col_name).strip()
if col_str in ignore_cols:
print(f" 跳过 '{col_name}': 系统保留列")
continue
y_series = data[col_name]
# 过滤非数值类型列 (避免将纯文本备注等拿去回归)
if not pd.api.types.is_numeric_dtype(y_series):
print(f" 跳过目标列 '{col_name}': 非数值类型")
print(f" 跳过 '{col_name}': 非数值类型")
continue
if y_series.isna().all():
print(f" 跳过 '{col_name}': 所有值为空")
continue
# 检查是否有非空值
if not y_series.isna().all():
y_dict[col_name] = y_series
print(f" 目标列 '{col_name}': {y_series.count()} 个非空值, 范围: {y_series.min():.4f} ~ {y_series.max():.4f}")
else:
print(f" 跳过目标列 '{col_name}': 所有值为空")
y_dict[col_name] = y_series
print(f" 目标列 '{col_name}': {y_series.count()} 个非空值, "
f"范围: {y_series.min():.4f} ~ {y_series.max():.4f}")
print(f"特征数据形状: {X.shape}")
print(f"有效目标列数量: {len(y_dict)}")
return X, y_dict
def load_data_single(self, csv_path: str, target_column_name: str, feature_start_column: Union[int, str]) -> Tuple[pd.DataFrame, pd.Series]:
def load_data_single(self, csv_path: str, target_column_name: str,
feature_start_column: Union[int, str, None] = None
) -> Tuple[pd.DataFrame, pd.Series]:
"""
加载单个目标列的CSV数据
加载单个目标列的CSV数据(v2:基于列名语义提取特征)
Args:
csv_path: CSV文件路径
target_column_name: 目标列名
feature_start_column: 特征开始列索引(int)或列名(str)
feature_start_column: 特征起始位置(可选,None=智能识别)
Returns:
X: 特征数据
X: 特征数据 (DataFrame,保留列名)
y: 目标值数据
"""
data = pd.read_csv(csv_path)
# 检查目标列是否存在
if target_column_name not in data.columns:
raise ValueError(f"目标列 '{target_column_name}' 不存在于数据中")
# 如果feature_start_column是列名,转换为索引
if isinstance(feature_start_column, str):
if feature_start_column not in data.columns:
raise ValueError(f"指定的特征开始列 '{feature_start_column}' 不存在于数据中")
feature_start_index = data.columns.get_loc(feature_start_column)
else:
feature_start_index = feature_start_column
# 提取目标值和特征
# 提取目标值
y = data[target_column_name]
X = data.iloc[:, feature_start_index:]
# 去除y值为空的行
# 去除 y 值为空的行
mask = ~y.isna()
data_cleaned = data[mask]
# 重新定义y和X,去除对应的空值行
y = data_cleaned[target_column_name]
X = data_cleaned.iloc[:, feature_start_column:]
# 提取特征(基于列名语义,排除目标列自身)
X, _ = self._extract_feature_columns(
data_cleaned, feature_start_column
)
# 确保目标列不在 X 中
if target_column_name in X.columns:
X = X.drop(columns=[target_column_name])
print(f"目标列 '{target_column_name}' 数据加载完成:")
print(f" 样本数量: {X.shape[0]}")
@ -744,6 +842,13 @@ class WaterQualityModelingBatch:
)
if result is not None:
# 提取训练波长列表(写入 metadata,供推理端光谱重采样)
train_wavelengths = self._extract_train_wavelengths(
X_raw.columns
)
print(f"[波长记忆] 提取到 {len(train_wavelengths)} 个训练波长: "
f"{train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm")
# 保存模型(result['model'] 已是 sklearn Pipeline)
metadata = {
'target_column_name': target_column_name,
@ -761,9 +866,11 @@ class WaterQualityModelingBatch:
'train_size': result['train_size'],
'test_size': result['test_size'],
'split_method': result['split_method'],
# Pipeline 标记(便于旧 inference 路径兼容/诊断)
'preprocess_method': preprocess_method,
'is_pipeline': result.get('is_pipeline', False),
# ★ 波长记忆:推理端可据此进行跨传感器光谱重采样
'train_wavelengths': train_wavelengths,
'train_columns': list(X_raw.columns),
}
self.save_model(result['model'], target_column_name,

View File

@ -377,110 +377,253 @@ class WaterQualityInference:
def preprocess_spectra(self, spectra: pd.DataFrame) -> np.ndarray:
"""
对光谱数据进行预处理
对光谱数据进行预处理 + 跨传感器光谱重采样。
改造要点(v2 — 光谱重采样):
- 废除原有的「列位置截断」和「零值填充」hack
- 从 model metadata 中读取训练时的波长列表
- 如果输入传感器与训练传感器波长网格不同,使用 np.interp()
将输入光谱重采样到训练波长网格
- 非光谱列(WQI 指数等)保持不变
- 兼容旧模型(无 train_wavelengths 时回退到旧逻辑)
Args:
spectra: 原始光谱数据
spectra: 原始光谱数据(DataFrame,列名为波长字符串)
Returns:
预处理后的光谱数据
预处理后的光谱数据 (numpy ndarray)
"""
if self.loaded_model_data is None:
raise ValueError("请先加载模型")
preprocess_method = self.loaded_model_data['preprocess_method']
# 处理 nan/NaN/None 值,转换为 "None" 字符串
# 处理 nan/NaN/None 值
if pd.isna(preprocess_method) or str(preprocess_method).lower() in ['nan', 'none', '']:
preprocess_method = "None"
# 解析预处理方法(可能包含划分方法前缀)
if '_' in str(preprocess_method):
parts = str(preprocess_method).split('_')
# 假设格式为 split_method_preprocess_method
actual_preprocess_method = '_'.join(parts[1:]) if len(parts) > 1 else parts[-1]
else:
actual_preprocess_method = str(preprocess_method)
# 再次检查并转换 nan
if actual_preprocess_method.lower() in ['nan', 'none', '']:
actual_preprocess_method = "None"
print(f"正在应用预处理方法: {actual_preprocess_method}")
print(f"原始光谱数据形状: {spectra.shape}")
# ---- 自动特征补全:50 光谱 → 补全至模型训练时的 95 维(WQI 指数) ----
# 触发条件:模型期望 n_features_in_ 个特征,但当前 spectra 列数不足
# 原因:training_spectra.csv 含 50 光谱 + 45 WQI;sampling_spectra.csv 只有 50 光谱
# 做法:与训练端(calculate_all_indices)完全一致的算法列表,实时补全缺失的 45 个 WQI 列
model = self.loaded_model_data['model']
expected_features = getattr(model, 'n_features_in_', None)
metadata = self.loaded_model_data.get('metadata', {})
train_wavelengths = metadata.get('train_wavelengths', None)
train_columns = metadata.get('train_columns', None)
# ---- 自动特征补全:50 光谱 → 补全至模型训练时的 n_features_in_ 维(WQI 指数) ----
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 < 模型期望 {expected_features} 列,"
f"正在从光谱数据实时计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
# ═══════════════════════════════════════════════════════════
# ★ 核心:光谱重采样(跨传感器适配)
# ═══════════════════════════════════════════════════════════
if train_wavelengths is not None and len(train_wavelengths) > 0:
print(f"[光谱重采样] 模型训练波长: {len(train_wavelengths)} 个, "
f"范围 {train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm")
# CSV 驱动的 WaterQualityIndexCalculator:所有公式名通过 list_available() 拿;
# 一次性 calculate_many() 批量计算。彻底摆脱 dir(calc) 反射扫描 + 单 algo_func
# 调用这种碎片化写法(Calculator 早已重构为公式驱动,不再有独立公式方法)。
formulas = calc.list_available()
if not formulas:
print("[特征补全] Calculator 未持有任何公式,跳过补全")
# 1) 从当前 spectra 中分离光谱列和 WQI 列
target_wavelengths = []
spectral_col_indices = []
wqi_col_indices = []
wqi_col_names = []
for i, col in enumerate(spectra.columns):
col_str = str(col)
try:
wl = float(col_str)
target_wavelengths.append(wl)
spectral_col_indices.append(i)
except (ValueError, TypeError):
if col_str.startswith('WQI_'):
wqi_col_indices.append(i)
wqi_col_names.append(col_str)
print(f"[光谱重采样] 当前传感器: {len(target_wavelengths)} 个波长列, "
f"{len(wqi_col_indices)} 个 WQI 列")
if len(target_wavelengths) == 0:
print("[光谱重采样] ⚠ 未检测到波长列,跳过重采样")
else:
# 2) 提取光谱数据矩阵
spectral_data = spectra.iloc[:, spectral_col_indices].values.astype(np.float64)
# 3) 逐行 np.interp 重采样
train_wl_arr = np.array(train_wavelengths, dtype=np.float64)
target_wl_arr = np.array(target_wavelengths, dtype=np.float64)
print(f"[光谱重采样] 执行重采样: {len(target_wavelengths)} → "
f"{len(train_wavelengths)} 个波长点 ...")
resampled = np.zeros((spectral_data.shape[0], len(train_wavelengths)),
dtype=np.float64)
for i in range(spectral_data.shape[0]):
resampled[i] = np.interp(
train_wl_arr, target_wl_arr, spectral_data[i],
left=np.nan, right=np.nan
)
# NaN 填充(外推区域):用最近有效值填充
nan_mask = np.isnan(resampled)
if nan_mask.any():
print(f"[光谱重采样] {nan_mask.sum()} 个 NaN (外推区域) → 最近邻填充")
for i in range(resampled.shape[0]):
row_nan = np.isnan(resampled[i])
if row_nan.any():
valid = ~row_nan
if valid.any():
valid_idx = np.where(valid)[0]
resampled[i, row_nan] = np.interp(
np.where(row_nan)[0], valid_idx,
resampled[i, valid]
)
else:
resampled[i] = 0.0
# 4) 重组为 DataFrame(列名 = 训练波长字符串)
wl_col_names = [f"{wl:.6f}" for wl in train_wavelengths]
resampled_df = pd.DataFrame(resampled, columns=wl_col_names,
index=spectra.index)
# 5) 拼接 WQI 列(如有)
if wqi_col_indices:
wqi_df = spectra.iloc[:, wqi_col_indices].copy()
wqi_df.columns = wqi_col_names
spectra = pd.concat([resampled_df, wqi_df], axis=1)
else:
# fast=True: Step9 特征补全走向量化快车道(~63 公式,已验证稳定)
results_df = calc.calculate_many(formulas, spectra, fast=True)
# results_df 是列对齐的 WQI 计算结果(每列一个公式,行数=样本数)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!光谱列已扩充至 {spectra.shape[1]} 列"
f"(追加了 {spectra.shape[1] - original_col_count} 个 WQI 指数)")
except Exception as e:
print(f"[特征补全] 失败,将使用原始光谱特征: {e}")
spectra = resampled_df
# ---- 防线 1:强制维度对齐(物理截断)----
if expected_features is not None and spectra.shape[1] > expected_features:
print(f"[精准对齐] 正在将 {spectra.shape[1]} 维特征截断为模型要求的 {expected_features} 维")
spectra = spectra.iloc[:, :expected_features]
elif expected_features is not None and spectra.shape[1] < expected_features:
# 维度不足时填充 0
padding_cols = expected_features - spectra.shape[1]
for i in range(padding_cols):
spectra[f'_padding_{i}'] = 0.0
print(f"[精准对齐] 特征不足,填充 {padding_cols} 列 0")
print(f"[光谱重采样] 完成: {spectra.shape[1]} 列 "
f"(光谱 {len(train_wavelengths)} + WQI {len(wqi_col_indices)})")
# ---- 防线 2:彻底清洗无穷大数值----
# 防止 WQI 计算中除零/溢出产生 np.inf / -np.inf 导致预处理崩溃
else:
# ── 兼容旧模型(无 train_wavelengths)──
print("[光谱重采样] 模型无 train_wavelengths 元数据(旧模型),"
"跳过光谱重采样,进入兼容路径...")
# 旧兼容逻辑:WQI 自动补全
expected_features = getattr(model, 'n_features_in_', None)
if expected_features is not None and spectra.shape[1] < expected_features:
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
try:
from src.utils.water_index import WaterQualityIndexCalculator
calc = WaterQualityIndexCalculator()
formulas = calc.list_available()
if formulas:
results_df = calc.calculate_many(formulas, spectra, fast=True)
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1]
spectra = pd.concat([spectra, results_df], axis=1)
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
f"(+{spectra.shape[1] - original_col_count} WQI)")
except Exception as e:
print(f"[特征补全] 失败: {e}")
# ★ v4: 无 train_wavelengths 时,基于物理波长的经验重采样
# 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格
# 取代原基于列索引的硬截断(不同传感器列索引对应的波长完全不同)
if expected_features is not None:
n_current = spectra.shape[1]
# ── 第1步:从列名中提取波长列和非波长列 ──
_target_wl: List[float] = []
_wqi_names: List[str] = []
_other_names: List[str] = []
for _col in spectra.columns:
try:
_target_wl.append(float(str(_col)))
except (ValueError, TypeError):
if str(_col).startswith('WQI_'):
_wqi_names.append(str(_col))
else:
_other_names.append(str(_col))
# 分离出 WQI 列(保存以便后续拼接回去)
_wqi_df = spectra[_wqi_names].copy() if _wqi_names else pd.DataFrame()
if len(_target_wl) >= 2 and expected_features > 0:
# ── 第2步:有真实波长 → np.interp 重采样 ──
fallback_wl = np.linspace(400.0, 800.0, expected_features,
dtype=np.float64)
_target_arr = np.array(_target_wl, dtype=np.float64)
_spec_data = spectra.iloc[:, :len(_target_wl)].values.astype(np.float64)
# 按波长排序(以防列顺序不是严格递增)
_sort_idx = np.argsort(_target_arr)
_target_arr = _target_arr[_sort_idx]
_spec_data = _spec_data[:, _sort_idx]
_resampled = np.zeros((_spec_data.shape[0], expected_features),
dtype=np.float64)
for i in range(_spec_data.shape[0]):
_resampled[i] = np.interp(
fallback_wl, _target_arr, _spec_data[i],
left=np.nan, right=np.nan
)
# NaN 填充
_nan = np.isnan(_resampled)
if _nan.any():
for i in range(_resampled.shape[0]):
row_nan = np.isnan(_resampled[i])
if row_nan.any():
valid = ~row_nan
if valid.any():
vi = np.where(valid)[0]
_resampled[i, row_nan] = np.interp(
np.where(row_nan)[0], vi, _resampled[i, valid]
)
else:
_resampled[i] = 0.0
# 重组 DataFrame
_wl_cols = [f"{w:.1f}" for w in fallback_wl]
spectra = pd.DataFrame(_resampled, columns=_wl_cols,
index=spectra.index)
if not _wqi_df.empty:
spectra = pd.concat([spectra, _wqi_df.reset_index(drop=True)],
axis=1)
print(f"[兼容重采样] 列名解析到 {len(_target_wl)} 个波长 "
f"({_target_arr[0]:.1f}~{_target_arr[-1]:.1f}nm) → "
f"np.interp → {expected_features} 列 (400~800nm 标准网格)"
f"{' + ' + str(len(_wqi_names)) + ' WQI' if _wqi_names else ''}")
elif n_current > expected_features:
# ── 第3步:无法解析波长(纯字符串列名)→ 硬截断兜底 ──
spectra = spectra.iloc[:, :expected_features]
print(f"[兼容截断] 无法解析波长列名,硬截断 "
f"{n_current} → {expected_features} 列")
elif n_current < expected_features:
# ── 第4步:特征不够 → 补零 ──
for i in range(expected_features - n_current):
spectra[f'_padding_{i}'] = 0.0
print(f"[兼容填充] 特征不足,补零 "
f"{n_current} → {expected_features} 列")
# ═══════════════════════════════════════════════════════════
# 通用清洗
# ═══════════════════════════════════════════════════════════
spectra = spectra.replace([np.inf, -np.inf], np.nan)
spectra = spectra.fillna(0)
print(f"[特征对齐] 最终输入维度: {spectra.shape}")
# ---- Pipeline 化分支:模型内置 scaler/MSC.mean_spectrum_ 等状态时,跳过手动 Preprocessing ----
# ── Pipeline 化分支:模型内置 scaler → 跳过手动 Preprocessing ──
if isinstance(model, Pipeline):
print(f"[Pipeline] 检测到模型是 sklearn Pipeline,"
f"其内置预处理步骤({list(model.named_steps.keys())[0]})将处理原始光谱,"
print(f"[Pipeline] 模型是 sklearn Pipeline,内置预处理步骤,"
f"无需外部 Preprocessing")
return spectra.values
# ---- 兼容路径:旧 .joblib(裸模型 + preprocess_method 字符串)回退手动 Preprocessing ----
# ── 兼容路径:旧裸模型 + 手动 Preprocessing ──
try:
# 应用预处理
spectra_processed = Preprocessing(actual_preprocess_method, spectra)
# 确保返回numpy数组
if isinstance(spectra_processed, pd.DataFrame):
spectra_processed = spectra_processed.values
print(f" [Legacy] 旧裸模型 + 手动 Preprocessing({actual_preprocess_method}) 完成,"
f"数据形状: {spectra_processed.shape}")
return spectra_processed
except Exception as e:
print(f"预处理失败: {e}")
print("使用原始数据")

View File

@ -129,7 +129,7 @@ class ModelingStep:
@staticmethod
def train_models(
feature_start_column: str = "374.285004",
feature_start_column: Union[int, str, None] = None,
preprocessing_methods: Optional[List[str]] = None,
model_names: Optional[List[str]] = None,
split_methods: Optional[List[str]] = None,

View File

@ -528,16 +528,14 @@ class Step8MlTrainPanel(QWidget):
def get_training_params(self):
"""获取模型训练参数"""
# ★ 2026-07-01 安全保护:float() 前验证 currentText() 是否为有效数值
# ★ v2:currentText() 为列名字符串,传 None 走智能识别
feature_text = self.feature_start.currentText()
try:
feature_start = float(feature_text)
except (ValueError, TypeError):
feature_start = 374.285004 # 默认光谱起始列
if not feature_text or feature_text.startswith("("):
feature_text = None
return {
'pipeline_type': 'machine_learning',
'feature_start': feature_start,
'feature_start': feature_text,
'cv_folds': self.cv_folds.value(),
'preprocess_methods': [method for method, cb in self.preproc_checkboxes.items() if cb.isChecked()],
'model_types': [model for model, cb in self.model_checkboxes.items() if cb.isChecked()],

View File

@ -14,7 +14,7 @@ Step8 后端计算服务(机器学习建模训练)
execute_step8({
"training_csv_path": "D:/training_spectra_indices.csv", # 训练 CSV(必填)
"feature_start_column": "374.285004", # 特征起始列名/索引
"feature_start_column": None, # None=智能识别特征列
"preprocessing_methods": ["None", "MMS"], # 预处理方法列表
"model_names": ["RF", "SVR", "Ridge", "Lasso"], # 模型列表
"split_methods": ["spxy"], # 划分方法列表
@ -68,7 +68,7 @@ def execute_step8(config: Dict[str, Any]) -> Dict[str, Any]:
"""
# ---------- 入参规整 ----------
training_csv_path: Optional[str] = config.get("training_csv_path")
feature_start_column: str = str(config.get("feature_start_column", "374.285004"))
feature_start_column: Optional[str] = config.get("feature_start_column", None)
preprocessing_methods: Optional[List[str]] = config.get("preprocessing_methods")
model_names: Optional[List[str]] = config.get("model_names")
split_methods: Optional[List[str]] = config.get("split_methods")

View File

@ -104,12 +104,12 @@ class Step8View(BaseView):
params_layout = QFormLayout()
self.feature_start = QLineEdit()
self.feature_start.setText("374.285004")
self.feature_start.setPlaceholderText("留空=智能识别波长列和WQI列")
params_layout.addRow("特征起始列:", self.feature_start)
feature_start_hint = QLabel(
"提示:请使用记事本打开 training_spectra.csv 确认首个波长的精确表头名称"
"(如 374.285 或 374.285004)并在此填入,避免因浮点精度差异导致列名匹配失败。"
"提示:留空则自动识别所有波长列(纯数字列名)和 WQI 指数列作为特征。\n"
"如需手动指定,可填入首个波长列名(如 374.285004)或列索引。"
)
feature_start_hint.setWordWrap(True)
feature_start_hint.setStyleSheet("color: #666; font-size: 10px;")
@ -253,7 +253,7 @@ class Step8View(BaseView):
]
config = {
"feature_start_column": self.feature_start.text(),
"feature_start_column": self.feature_start.text().strip() or None,
"preprocessing_methods": preprocessing_methods if preprocessing_methods else ["None"],
"model_names": model_names if model_names else ["SVR"],
"split_methods": split_methods if split_methods else ["random"],
@ -270,7 +270,8 @@ class Step8View(BaseView):
def set_config(self, config: dict):
if "feature_start_column" in config:
self.feature_start.setText(str(config["feature_start_column"]))
val = config["feature_start_column"]
self.feature_start.setText(str(val) if val is not None else "")
if "cv_folds" in config:
self.cv_folds.setValue(config["cv_folds"])
if "preprocessing_methods" in config: