From 92e8c9037048d6725eb9077b448eb1f4a788d6ae Mon Sep 17 00:00:00 2001 From: duxin Date: Mon, 27 Jul 2026 14:52:30 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20ML=E6=A8=A1=E5=9E=8B=E6=B3=A2=E9=95=BF?= =?UTF-8?q?=E8=AE=B0=E5=BF=86=20+=20=E8=B7=A8=E4=BC=A0=E6=84=9F=E5=99=A8?= =?UTF-8?q?=E5=85=89=E8=B0=B1=E9=87=8D=E9=87=87=E6=A0=B7=20+=20=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=8A=A0=E8=BD=BD=E8=A7=84=E8=8C=83=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 训练端 (modeling_batch.py): - _extract_train_wavelengths() 从列名提取训练波长写入 metadata - load_data_batch/load_data_single 改为基于列名语义智能提取特征 - 废除 feature_start_column 硬编码位置索引 推理端 (inference_batch.py): - preprocess_spectra() 分支A: train_wavelengths存在→np.interp精确重采样 - preprocess_spectra() 分支B: 无波长元数据→解析列名→np.interp到400-800nm标准网格 - 废除暴力截断和零值填充 --- src/core/handlers/step8_ml_train.py | 2 +- src/core/modeling/modeling_batch.py | 255 +++++++++++++++++------- src/core/prediction/inference_batch.py | 263 +++++++++++++++++++------ src/core/steps/modeling_step.py | 2 +- src/gui/panels/step8_ml_train_panel.py | 10 +- src/new/services/step8_service.py | 4 +- src/new/views/step8_view.py | 11 +- 7 files changed, 398 insertions(+), 149 deletions(-) diff --git a/src/core/handlers/step8_ml_train.py b/src/core/handlers/step8_ml_train.py index 4a291b2..9d8e742 100644 --- a/src/core/handlers/step8_ml_train.py +++ b/src/core/handlers/step8_ml_train.py @@ -32,7 +32,7 @@ class Step8MlTrainHandler(BaseStepHandler): try: result = ModelingStep.train_models( - feature_start_column=config.get('feature_start_column', '374.285004'), + feature_start_column=config.get('feature_start_column', None), preprocessing_methods=config.get('preprocessing_methods'), model_names=config.get('model_names'), split_methods=config.get('split_methods'), diff --git a/src/core/modeling/modeling_batch.py b/src/core/modeling/modeling_batch.py index 6d70b2c..ceac44d 100644 --- a/src/core/modeling/modeling_batch.py +++ b/src/core/modeling/modeling_batch.py @@ -231,19 +231,125 @@ class WaterQualityModelingBatch: self.results = {} self.best_models = {} - def load_data_batch(self, csv_path: str, feature_start_column: Union[int, str]) -> Tuple[pd.DataFrame, Dict[str, pd.Series]]: - """ - 批量加载CSV数据，将指定列之前的列作为目标值 + @staticmethod + def _is_wavelength_column(col_name: str) -> bool: + """判断列名是否为波长值（纯数字字符串，如 '374.285004'）""" + try: + float(str(col_name)) + return True + except (ValueError, TypeError): + return False - Args: - csv_path: CSV文件路径 - feature_start_column: 特征开始列索引（int）或列名（str） + @staticmethod + def _is_wqi_column(col_name: str) -> bool: + """判断列名是否为 WQI 水质指数列（'WQI_' 前缀）""" + return str(col_name).startswith('WQI_') + + @staticmethod + def _extract_train_wavelengths(columns) -> List[float]: + """从列名列表中提取波长值（float 列表） + + 遍历列名，将所有可转为 float 的列名提取为波长列表。 + 用于写入模型 metadata['train_wavelengths']，供推理端光谱重采样。 + """ + wl_list = [] + for c in columns: + try: + wl_list.append(float(str(c))) + except (ValueError, TypeError): + pass + return wl_list + + def _extract_feature_columns(self, data: pd.DataFrame, + feature_start_column: Union[int, str, None] = None + ) -> Tuple[pd.DataFrame, List[int]]: + """从 DataFrame 中提取特征列（基于列名语义，而非位置索引） + + 策略（优先级从高到低）： + 1) 如果 feature_start_column 是列名（str），定位该列并取之后的列 + 2) 如果 feature_start_column 是整数，兼容旧逻辑（位置索引） + 3) 如果 feature_start_column 为 None，自动识别： + - 保留所有纯数字列名（波长列） + - 保留所有 WQI_ 前缀列（水质指数列） + - 跳过坐标/元数据列 Returns: - X: 特征数据 + X: 特征 DataFrame（保留列名） + feature_col_indices: 特征列在原 data 中的位置索引列表 + """ + all_cols = list(data.columns) + + if feature_start_column is not None: + # ── 兼容旧逻辑：按列名或索引位置截取 ── + if isinstance(feature_start_column, str): + if feature_start_column not in data.columns: + raise ValueError( + f"指定的特征开始列 '{feature_start_column}' 不存在于数据中" + ) + start_idx = data.columns.get_loc(feature_start_column) + print(f"[特征提取] 按列名 '{feature_start_column}' 定位 → 索引 {start_idx}") + else: + start_idx = int(feature_start_column) + print(f"[特征提取] 按位置索引 {start_idx} 截取") + + X = data.iloc[:, start_idx:] + feature_indices = list(range(start_idx, len(all_cols))) + else: + # ── 智能识别：按列名语义过滤 ── + # 黑名单：坐标列、元数据列（不会被误判为特征） + _meta_patterns = { + 'x_coord', 'y_coord', 'pixel_x', 'pixel_y', + 'longitude', 'latitude', 'lon', 'lat', + 'id', 'station', 'sample_id', + } + feature_indices = [] + for i, col in enumerate(all_cols): + col_lower = str(col).lower().strip() + # 跳过元数据列 + if col_lower in _meta_patterns: + continue + # 保留波长列 + if self._is_wavelength_column(col): + feature_indices.append(i) + # 保留 WQI 列 + elif self._is_wqi_column(col): + feature_indices.append(i) + # 其他：跳过（可能是目标列或其他非特征列） + + if not feature_indices: + raise ValueError( + "智能特征提取失败：未找到任何波长列或 WQI 列。" + f"CSV 列名: {all_cols[:10]}..." + ) + X = data.iloc[:, feature_indices] + print(f"[特征提取] 智能识别: {len(feature_indices)} 个特征列 " + f"(波长列 + WQI 列)") + + print(f"[特征提取] 特征数据形状: {X.shape}") + return X, feature_indices + + def load_data_batch(self, csv_path: str, + feature_start_column: Union[int, str, None] = None + ) -> Tuple[pd.DataFrame, Dict[str, pd.Series]]: + """批量加载 CSV 数据，自动识别特征列与目标列 + + 改造要点（v2）： + - 特征列按列名语义提取（波长数字 / WQI_ 前缀），不再按硬编码位置一刀切 + - 目标列 = 不在特征列中、且非系统保留列（ID/坐标等）的数值列 + - X 保持为 DataFrame，列名保留波长信息（供后续提取 train_wavelengths） + + Args: + csv_path: CSV 文件路径 + feature_start_column: （可选）旧版兼容参数： + - str: 特征起始列名 + - int: 特征起始列索引 + - None: 自动智能识别 + + Returns: + X: 特征数据 (DataFrame，保留列名) y_dict: 目标值数据字典，键为列名 """ - # 读取CSV数据，处理空字符串和缺失值 + # 读取 CSV 数据，处理空字符串和缺失值 try: data = pd.read_csv(csv_path, na_values=['', ' ', 'NaN', 'nan', 'NULL', 'null']) except pd.errors.EmptyDataError: @@ -251,113 +357,105 @@ class WaterQualityModelingBatch: except Exception as e: raise ValueError(f"读取CSV文件 '{csv_path}' 时出错: {e}") - # 检查并清理数据中的空字符串和其他无效值 print("数据清理...") original_shape = data.shape - # 将空字符串替换为NaN + # 将空字符串替换为 NaN data = data.replace(r'^\s*$', np.nan, regex=True) - # 对于数值列，将无法转换为数字的字符串替换为NaN + # 对于数值列，将无法转换为数字的字符串替换为 NaN for col in data.columns: try: - # 尝试将列转换为数值类型 data[col] = pd.to_numeric(data[col], errors='coerce') except Exception: - # 如果转换失败，保持原样（可能是字符串列） pass cleaned_shape = data.shape if cleaned_shape != original_shape: - print(f"数据清理完成: {original_shape[0]}行{original_shape[1]}列 -> {cleaned_shape[0]}行{cleaned_shape[1]}列") - + print(f"数据清理完成: {original_shape[0]}行{original_shape[1]}列 " + f"-> {cleaned_shape[0]}行{cleaned_shape[1]}列") + print(f"数据加载完成，总列数: {data.shape[1]}") - print(f"所有列名: {list(data.columns)}") - - # 如果feature_start_column是列名，转换为索引 - if isinstance(feature_start_column, str): - if feature_start_column not in data.columns: - raise ValueError(f"指定的特征开始列 '{feature_start_column}' 不存在于数据中") - feature_start_index = data.columns.get_loc(feature_start_column) - print(f"特征开始列 '{feature_start_column}' 对应索引: {feature_start_index}") - else: - feature_start_index = feature_start_column - print(f"特征开始列索引: {feature_start_index}") - - # 提取特征数据（从feature_start_index开始） - X = data.iloc[:, feature_start_index:] - - # 提取所有目标列（从0列到feature_start_index-1列） + print(f"所有列名: {list(data.columns)[:10]}..." if len(data.columns) > 10 + else f"所有列名: {list(data.columns)}") + + # ── 提取特征列（基于列名语义） ── + X, feature_indices = self._extract_feature_columns( + data, feature_start_column + ) + + # ── 提取目标列（不在特征列中 + 非系统保留列 + 数值类型） ── + feature_idx_set = set(feature_indices) + ignore_cols = { + 'ID', 'id', 'Id', + 'Longitude', 'Latitude', 'Lon', 'Lat', + 'longitude', 'latitude', 'lon', 'lat', + 'Station', 'station', 'sample_id', + 'x_coord', 'y_coord', 'pixel_x', 'pixel_y', + } + y_dict = {} - target_columns = data.columns[:feature_start_index] - print(f"检测到的潜在目标列: {list(target_columns)}") - - # 新增：跳过非预测目标的系统保留列 - ignore_cols = {'ID', 'id', 'Id', 'Longitude', 'Latitude', 'Lon', 'Lat', 'longitude', 'latitude', 'lon', 'lat', 'Station', 'station'} - - for col_name in target_columns: - # 过滤黑名单列 - if col_name in ignore_cols: - print(f" 跳过目标列 '{col_name}': 属于系统保留列或空间坐标") + for i, col_name in enumerate(data.columns): + if i in feature_idx_set: + continue # 已在特征矩阵中 + col_str = str(col_name).strip() + if col_str in ignore_cols: + print(f" 跳过 '{col_name}': 系统保留列") continue y_series = data[col_name] - - # 过滤非数值类型列 (避免将纯文本备注等拿去回归) if not pd.api.types.is_numeric_dtype(y_series): - print(f" 跳过目标列 '{col_name}': 非数值类型") + print(f" 跳过 '{col_name}': 非数值类型") + continue + if y_series.isna().all(): + print(f" 跳过 '{col_name}': 所有值为空") continue - # 检查是否有非空值 - if not y_series.isna().all(): - y_dict[col_name] = y_series - print(f" 目标列 '{col_name}': {y_series.count()} 个非空值, 范围: {y_series.min():.4f} ~ {y_series.max():.4f}") - else: - print(f" 跳过目标列 '{col_name}': 所有值为空") - + y_dict[col_name] = y_series + print(f" 目标列 '{col_name}': {y_series.count()} 个非空值, " + f"范围: {y_series.min():.4f} ~ {y_series.max():.4f}") + print(f"特征数据形状: {X.shape}") print(f"有效目标列数量: {len(y_dict)}") - + return X, y_dict - def load_data_single(self, csv_path: str, target_column_name: str, feature_start_column: Union[int, str]) -> Tuple[pd.DataFrame, pd.Series]: + def load_data_single(self, csv_path: str, target_column_name: str, + feature_start_column: Union[int, str, None] = None + ) -> Tuple[pd.DataFrame, pd.Series]: """ - 加载单个目标列的CSV数据 + 加载单个目标列的CSV数据（v2：基于列名语义提取特征） Args: csv_path: CSV文件路径 target_column_name: 目标列名 - feature_start_column: 特征开始列索引（int）或列名（str） + feature_start_column: 特征起始位置（可选，None=智能识别） Returns: - X: 特征数据 + X: 特征数据 (DataFrame，保留列名) y: 目标值数据 """ data = pd.read_csv(csv_path) - + # 检查目标列是否存在 if target_column_name not in data.columns: raise ValueError(f"目标列 '{target_column_name}' 不存在于数据中") - - # 如果feature_start_column是列名，转换为索引 - if isinstance(feature_start_column, str): - if feature_start_column not in data.columns: - raise ValueError(f"指定的特征开始列 '{feature_start_column}' 不存在于数据中") - feature_start_index = data.columns.get_loc(feature_start_column) - else: - feature_start_index = feature_start_column - - # 提取目标值和特征 + + # 提取目标值 y = data[target_column_name] - X = data.iloc[:, feature_start_index:] - - # 去除y值为空的行 + + # 去除 y 值为空的行 mask = ~y.isna() data_cleaned = data[mask] - - # 重新定义y和X，去除对应的空值行 y = data_cleaned[target_column_name] - X = data_cleaned.iloc[:, feature_start_column:] + + # 提取特征（基于列名语义，排除目标列自身） + X, _ = self._extract_feature_columns( + data_cleaned, feature_start_column + ) + # 确保目标列不在 X 中 + if target_column_name in X.columns: + X = X.drop(columns=[target_column_name]) print(f"目标列 '{target_column_name}' 数据加载完成:") print(f" 样本数量: {X.shape[0]}") @@ -744,6 +842,13 @@ class WaterQualityModelingBatch: ) if result is not None: + # 提取训练波长列表（写入 metadata，供推理端光谱重采样） + train_wavelengths = self._extract_train_wavelengths( + X_raw.columns + ) + print(f"[波长记忆] 提取到 {len(train_wavelengths)} 个训练波长: " + f"{train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm") + # 保存模型（result['model'] 已是 sklearn Pipeline） metadata = { 'target_column_name': target_column_name, @@ -761,9 +866,11 @@ class WaterQualityModelingBatch: 'train_size': result['train_size'], 'test_size': result['test_size'], 'split_method': result['split_method'], - # Pipeline 标记（便于旧 inference 路径兼容/诊断） 'preprocess_method': preprocess_method, 'is_pipeline': result.get('is_pipeline', False), + # ★ 波长记忆：推理端可据此进行跨传感器光谱重采样 + 'train_wavelengths': train_wavelengths, + 'train_columns': list(X_raw.columns), } self.save_model(result['model'], target_column_name, diff --git a/src/core/prediction/inference_batch.py b/src/core/prediction/inference_batch.py index 7864a56..6fdd8ce 100644 --- a/src/core/prediction/inference_batch.py +++ b/src/core/prediction/inference_batch.py @@ -377,110 +377,253 @@ class WaterQualityInference: def preprocess_spectra(self, spectra: pd.DataFrame) -> np.ndarray: """ - 对光谱数据进行预处理 + 对光谱数据进行预处理 + 跨传感器光谱重采样。 + + 改造要点（v2 — 光谱重采样）： + - 废除原有的「列位置截断」和「零值填充」hack + - 从 model metadata 中读取训练时的波长列表 + - 如果输入传感器与训练传感器波长网格不同，使用 np.interp() + 将输入光谱重采样到训练波长网格 + - 非光谱列（WQI 指数等）保持不变 + - 兼容旧模型（无 train_wavelengths 时回退到旧逻辑） Args: - spectra: 原始光谱数据 + spectra: 原始光谱数据（DataFrame，列名为波长字符串） Returns: - 预处理后的光谱数据 + 预处理后的光谱数据 (numpy ndarray) """ if self.loaded_model_data is None: raise ValueError("请先加载模型") preprocess_method = self.loaded_model_data['preprocess_method'] - - # 处理 nan/NaN/None 值，转换为 "None" 字符串 + + # 处理 nan/NaN/None 值 if pd.isna(preprocess_method) or str(preprocess_method).lower() in ['nan', 'none', '']: preprocess_method = "None" - + # 解析预处理方法（可能包含划分方法前缀） if '_' in str(preprocess_method): parts = str(preprocess_method).split('_') - # 假设格式为 split_method_preprocess_method actual_preprocess_method = '_'.join(parts[1:]) if len(parts) > 1 else parts[-1] else: actual_preprocess_method = str(preprocess_method) - - # 再次检查并转换 nan + if actual_preprocess_method.lower() in ['nan', 'none', '']: actual_preprocess_method = "None" print(f"正在应用预处理方法: {actual_preprocess_method}") print(f"原始光谱数据形状: {spectra.shape}") - # ---- 自动特征补全：50 光谱 → 补全至模型训练时的 95 维（WQI 指数） ---- - # 触发条件：模型期望 n_features_in_ 个特征，但当前 spectra 列数不足 - # 原因：training_spectra.csv 含 50 光谱 + 45 WQI；sampling_spectra.csv 只有 50 光谱 - # 做法：与训练端（calculate_all_indices）完全一致的算法列表，实时补全缺失的 45 个 WQI 列 model = self.loaded_model_data['model'] - expected_features = getattr(model, 'n_features_in_', None) + metadata = self.loaded_model_data.get('metadata', {}) + train_wavelengths = metadata.get('train_wavelengths', None) + train_columns = metadata.get('train_columns', None) - # ---- 自动特征补全：50 光谱 → 补全至模型训练时的 n_features_in_ 维（WQI 指数） ---- - if expected_features is not None and spectra.shape[1] < expected_features: - print(f"[特征补全] 检测到特征缺口：当前 {spectra.shape[1]} 列 < 模型期望 {expected_features} 列，" - f"正在从光谱数据实时计算 WQI 指数...") - try: - from src.utils.water_index import WaterQualityIndexCalculator - calc = WaterQualityIndexCalculator() + # ═══════════════════════════════════════════════════════════ + # ★ 核心：光谱重采样（跨传感器适配） + # ═══════════════════════════════════════════════════════════ + if train_wavelengths is not None and len(train_wavelengths) > 0: + print(f"[光谱重采样] 模型训练波长: {len(train_wavelengths)} 个, " + f"范围 {train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm") - # CSV 驱动的 WaterQualityIndexCalculator：所有公式名通过 list_available() 拿； - # 一次性 calculate_many() 批量计算。彻底摆脱 dir(calc) 反射扫描 + 单 algo_func - # 调用这种碎片化写法（Calculator 早已重构为公式驱动，不再有独立公式方法）。 - formulas = calc.list_available() - if not formulas: - print("[特征补全] Calculator 未持有任何公式，跳过补全") + # 1) 从当前 spectra 中分离光谱列和 WQI 列 + target_wavelengths = [] + spectral_col_indices = [] + wqi_col_indices = [] + wqi_col_names = [] + + for i, col in enumerate(spectra.columns): + col_str = str(col) + try: + wl = float(col_str) + target_wavelengths.append(wl) + spectral_col_indices.append(i) + except (ValueError, TypeError): + if col_str.startswith('WQI_'): + wqi_col_indices.append(i) + wqi_col_names.append(col_str) + + print(f"[光谱重采样] 当前传感器: {len(target_wavelengths)} 个波长列, " + f"{len(wqi_col_indices)} 个 WQI 列") + + if len(target_wavelengths) == 0: + print("[光谱重采样] ⚠ 未检测到波长列，跳过重采样") + else: + # 2) 提取光谱数据矩阵 + spectral_data = spectra.iloc[:, spectral_col_indices].values.astype(np.float64) + + # 3) 逐行 np.interp 重采样 + train_wl_arr = np.array(train_wavelengths, dtype=np.float64) + target_wl_arr = np.array(target_wavelengths, dtype=np.float64) + + print(f"[光谱重采样] 执行重采样: {len(target_wavelengths)} → " + f"{len(train_wavelengths)} 个波长点 ...") + resampled = np.zeros((spectral_data.shape[0], len(train_wavelengths)), + dtype=np.float64) + for i in range(spectral_data.shape[0]): + resampled[i] = np.interp( + train_wl_arr, target_wl_arr, spectral_data[i], + left=np.nan, right=np.nan + ) + + # NaN 填充（外推区域）：用最近有效值填充 + nan_mask = np.isnan(resampled) + if nan_mask.any(): + print(f"[光谱重采样] {nan_mask.sum()} 个 NaN (外推区域) → 最近邻填充") + for i in range(resampled.shape[0]): + row_nan = np.isnan(resampled[i]) + if row_nan.any(): + valid = ~row_nan + if valid.any(): + valid_idx = np.where(valid)[0] + resampled[i, row_nan] = np.interp( + np.where(row_nan)[0], valid_idx, + resampled[i, valid] + ) + else: + resampled[i] = 0.0 + + # 4) 重组为 DataFrame（列名 = 训练波长字符串） + wl_col_names = [f"{wl:.6f}" for wl in train_wavelengths] + resampled_df = pd.DataFrame(resampled, columns=wl_col_names, + index=spectra.index) + + # 5) 拼接 WQI 列（如有） + if wqi_col_indices: + wqi_df = spectra.iloc[:, wqi_col_indices].copy() + wqi_df.columns = wqi_col_names + spectra = pd.concat([resampled_df, wqi_df], axis=1) else: - # fast=True: Step9 特征补全走向量化快车道（~63 公式，已验证稳定） - results_df = calc.calculate_many(formulas, spectra, fast=True) - # results_df 是列对齐的 WQI 计算结果（每列一个公式，行数=样本数） - if isinstance(results_df, pd.DataFrame) and not results_df.empty: - original_col_count = spectra.shape[1] - spectra = pd.concat([spectra, results_df], axis=1) - print(f"[特征补全] 完成！光谱列已扩充至 {spectra.shape[1]} 列" - f"（追加了 {spectra.shape[1] - original_col_count} 个 WQI 指数）") - except Exception as e: - print(f"[特征补全] 失败，将使用原始光谱特征: {e}") + spectra = resampled_df - # ---- 防线 1：强制维度对齐（物理截断）---- - if expected_features is not None and spectra.shape[1] > expected_features: - print(f"[精准对齐] 正在将 {spectra.shape[1]} 维特征截断为模型要求的 {expected_features} 维") - spectra = spectra.iloc[:, :expected_features] - elif expected_features is not None and spectra.shape[1] < expected_features: - # 维度不足时填充 0 - padding_cols = expected_features - spectra.shape[1] - for i in range(padding_cols): - spectra[f'_padding_{i}'] = 0.0 - print(f"[精准对齐] 特征不足，填充 {padding_cols} 列 0") + print(f"[光谱重采样] 完成: {spectra.shape[1]} 列 " + f"(光谱 {len(train_wavelengths)} + WQI {len(wqi_col_indices)})") - # ---- 防线 2：彻底清洗无穷大数值---- - # 防止 WQI 计算中除零/溢出产生 np.inf / -np.inf 导致预处理崩溃 + else: + # ── 兼容旧模型（无 train_wavelengths）── + print("[光谱重采样] 模型无 train_wavelengths 元数据（旧模型），" + "跳过光谱重采样，进入兼容路径...") + # 旧兼容逻辑：WQI 自动补全 + expected_features = getattr(model, 'n_features_in_', None) + if expected_features is not None and spectra.shape[1] < expected_features: + print(f"[特征补全] 检测到特征缺口：当前 {spectra.shape[1]} 列 " + f"< 模型期望 {expected_features} 列，正在计算 WQI 指数...") + try: + from src.utils.water_index import WaterQualityIndexCalculator + calc = WaterQualityIndexCalculator() + formulas = calc.list_available() + if formulas: + results_df = calc.calculate_many(formulas, spectra, fast=True) + if isinstance(results_df, pd.DataFrame) and not results_df.empty: + original_col_count = spectra.shape[1] + spectra = pd.concat([spectra, results_df], axis=1) + print(f"[特征补全] 完成！扩充至 {spectra.shape[1]} 列 " + f"(+{spectra.shape[1] - original_col_count} WQI)") + except Exception as e: + print(f"[特征补全] 失败: {e}") + + # ★ v4: 无 train_wavelengths 时，基于物理波长的经验重采样 + # 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格 + # 取代原基于列索引的硬截断（不同传感器列索引对应的波长完全不同） + if expected_features is not None: + n_current = spectra.shape[1] + + # ── 第1步：从列名中提取波长列和非波长列 ── + _target_wl: List[float] = [] + _wqi_names: List[str] = [] + _other_names: List[str] = [] + for _col in spectra.columns: + try: + _target_wl.append(float(str(_col))) + except (ValueError, TypeError): + if str(_col).startswith('WQI_'): + _wqi_names.append(str(_col)) + else: + _other_names.append(str(_col)) + + # 分离出 WQI 列（保存以便后续拼接回去） + _wqi_df = spectra[_wqi_names].copy() if _wqi_names else pd.DataFrame() + + if len(_target_wl) >= 2 and expected_features > 0: + # ── 第2步：有真实波长 → np.interp 重采样 ── + fallback_wl = np.linspace(400.0, 800.0, expected_features, + dtype=np.float64) + _target_arr = np.array(_target_wl, dtype=np.float64) + _spec_data = spectra.iloc[:, :len(_target_wl)].values.astype(np.float64) + # 按波长排序（以防列顺序不是严格递增） + _sort_idx = np.argsort(_target_arr) + _target_arr = _target_arr[_sort_idx] + _spec_data = _spec_data[:, _sort_idx] + + _resampled = np.zeros((_spec_data.shape[0], expected_features), + dtype=np.float64) + for i in range(_spec_data.shape[0]): + _resampled[i] = np.interp( + fallback_wl, _target_arr, _spec_data[i], + left=np.nan, right=np.nan + ) + # NaN 填充 + _nan = np.isnan(_resampled) + if _nan.any(): + for i in range(_resampled.shape[0]): + row_nan = np.isnan(_resampled[i]) + if row_nan.any(): + valid = ~row_nan + if valid.any(): + vi = np.where(valid)[0] + _resampled[i, row_nan] = np.interp( + np.where(row_nan)[0], vi, _resampled[i, valid] + ) + else: + _resampled[i] = 0.0 + + # 重组 DataFrame + _wl_cols = [f"{w:.1f}" for w in fallback_wl] + spectra = pd.DataFrame(_resampled, columns=_wl_cols, + index=spectra.index) + if not _wqi_df.empty: + spectra = pd.concat([spectra, _wqi_df.reset_index(drop=True)], + axis=1) + print(f"[兼容重采样] 列名解析到 {len(_target_wl)} 个波长 " + f"({_target_arr[0]:.1f}~{_target_arr[-1]:.1f}nm) → " + f"np.interp → {expected_features} 列 (400~800nm 标准网格)" + f"{' + ' + str(len(_wqi_names)) + ' WQI' if _wqi_names else ''}") + + elif n_current > expected_features: + # ── 第3步：无法解析波长（纯字符串列名）→ 硬截断兜底 ── + spectra = spectra.iloc[:, :expected_features] + print(f"[兼容截断] 无法解析波长列名，硬截断 " + f"{n_current} → {expected_features} 列") + + elif n_current < expected_features: + # ── 第4步：特征不够 → 补零 ── + for i in range(expected_features - n_current): + spectra[f'_padding_{i}'] = 0.0 + print(f"[兼容填充] 特征不足，补零 " + f"{n_current} → {expected_features} 列") + + # ═══════════════════════════════════════════════════════════ + # 通用清洗 + # ═══════════════════════════════════════════════════════════ spectra = spectra.replace([np.inf, -np.inf], np.nan) spectra = spectra.fillna(0) - print(f"[特征对齐] 最终输入维度: {spectra.shape}") - # ---- Pipeline 化分支：模型内置 scaler/MSC.mean_spectrum_ 等状态时，跳过手动 Preprocessing ---- + # ── Pipeline 化分支：模型内置 scaler → 跳过手动 Preprocessing ── if isinstance(model, Pipeline): - print(f"[Pipeline] 检测到模型是 sklearn Pipeline，" - f"其内置预处理步骤（{list(model.named_steps.keys())[0]}）将处理原始光谱，" + print(f"[Pipeline] 模型是 sklearn Pipeline，内置预处理步骤，" f"无需外部 Preprocessing") return spectra.values - # ---- 兼容路径：旧 .joblib（裸模型 + preprocess_method 字符串）回退手动 Preprocessing ---- + # ── 兼容路径：旧裸模型 + 手动 Preprocessing ── try: - # 应用预处理 spectra_processed = Preprocessing(actual_preprocess_method, spectra) - - # 确保返回numpy数组 if isinstance(spectra_processed, pd.DataFrame): spectra_processed = spectra_processed.values - print(f" [Legacy] 旧裸模型 + 手动 Preprocessing({actual_preprocess_method}) 完成，" f"数据形状: {spectra_processed.shape}") - return spectra_processed - except Exception as e: print(f"预处理失败: {e}") print("使用原始数据") diff --git a/src/core/steps/modeling_step.py b/src/core/steps/modeling_step.py index 0c19b90..9b26521 100644 --- a/src/core/steps/modeling_step.py +++ b/src/core/steps/modeling_step.py @@ -129,7 +129,7 @@ class ModelingStep: @staticmethod def train_models( - feature_start_column: str = "374.285004", + feature_start_column: Union[int, str, None] = None, preprocessing_methods: Optional[List[str]] = None, model_names: Optional[List[str]] = None, split_methods: Optional[List[str]] = None, diff --git a/src/gui/panels/step8_ml_train_panel.py b/src/gui/panels/step8_ml_train_panel.py index cba5a64..9a7f7ba 100644 --- a/src/gui/panels/step8_ml_train_panel.py +++ b/src/gui/panels/step8_ml_train_panel.py @@ -528,16 +528,14 @@ class Step8MlTrainPanel(QWidget): def get_training_params(self): """获取模型训练参数""" - # ★ 2026-07-01 安全保护：float() 前验证 currentText() 是否为有效数值 + # ★ v2：currentText() 为列名字符串，传 None 走智能识别 feature_text = self.feature_start.currentText() - try: - feature_start = float(feature_text) - except (ValueError, TypeError): - feature_start = 374.285004 # 默认光谱起始列 + if not feature_text or feature_text.startswith("（"): + feature_text = None return { 'pipeline_type': 'machine_learning', - 'feature_start': feature_start, + 'feature_start': feature_text, 'cv_folds': self.cv_folds.value(), 'preprocess_methods': [method for method, cb in self.preproc_checkboxes.items() if cb.isChecked()], 'model_types': [model for model, cb in self.model_checkboxes.items() if cb.isChecked()], diff --git a/src/new/services/step8_service.py b/src/new/services/step8_service.py index 3e5e1f5..4e5e5bc 100644 --- a/src/new/services/step8_service.py +++ b/src/new/services/step8_service.py @@ -14,7 +14,7 @@ Step8 后端计算服务（机器学习建模训练） execute_step8({ "training_csv_path": "D:/training_spectra_indices.csv", # 训练 CSV（必填） - "feature_start_column": "374.285004", # 特征起始列名/索引 + "feature_start_column": None, # None=智能识别特征列 "preprocessing_methods": ["None", "MMS"], # 预处理方法列表 "model_names": ["RF", "SVR", "Ridge", "Lasso"], # 模型列表 "split_methods": ["spxy"], # 划分方法列表 @@ -68,7 +68,7 @@ def execute_step8(config: Dict[str, Any]) -> Dict[str, Any]: """ # ---------- 入参规整 ---------- training_csv_path: Optional[str] = config.get("training_csv_path") - feature_start_column: str = str(config.get("feature_start_column", "374.285004")) + feature_start_column: Optional[str] = config.get("feature_start_column", None) preprocessing_methods: Optional[List[str]] = config.get("preprocessing_methods") model_names: Optional[List[str]] = config.get("model_names") split_methods: Optional[List[str]] = config.get("split_methods") diff --git a/src/new/views/step8_view.py b/src/new/views/step8_view.py index d067c6f..3314d22 100644 --- a/src/new/views/step8_view.py +++ b/src/new/views/step8_view.py @@ -104,12 +104,12 @@ class Step8View(BaseView): params_layout = QFormLayout() self.feature_start = QLineEdit() - self.feature_start.setText("374.285004") + self.feature_start.setPlaceholderText("留空=智能识别波长列和WQI列") params_layout.addRow("特征起始列:", self.feature_start) feature_start_hint = QLabel( - "提示：请使用记事本打开 training_spectra.csv 确认首个波长的精确表头名称" - "（如 374.285 或 374.285004）并在此填入，避免因浮点精度差异导致列名匹配失败。" + "提示：留空则自动识别所有波长列（纯数字列名）和 WQI 指数列作为特征。\n" + "如需手动指定，可填入首个波长列名（如 374.285004）或列索引。" ) feature_start_hint.setWordWrap(True) feature_start_hint.setStyleSheet("color: #666; font-size: 10px;") @@ -253,7 +253,7 @@ class Step8View(BaseView): ] config = { - "feature_start_column": self.feature_start.text(), + "feature_start_column": self.feature_start.text().strip() or None, "preprocessing_methods": preprocessing_methods if preprocessing_methods else ["None"], "model_names": model_names if model_names else ["SVR"], "split_methods": split_methods if split_methods else ["random"], @@ -270,7 +270,8 @@ class Step8View(BaseView): def set_config(self, config: dict): if "feature_start_column" in config: - self.feature_start.setText(str(config["feature_start_column"])) + val = config["feature_start_column"] + self.feature_start.setText(str(val) if val is not None else "") if "cv_folds" in config: self.cv_folds.setValue(config["cv_folds"]) if "preprocessing_methods" in config: