feat: ML模型波长记忆 + 跨传感器光谱重采样 + 数据加载规范化
训练端 (modeling_batch.py): - _extract_train_wavelengths() 从列名提取训练波长写入 metadata - load_data_batch/load_data_single 改为基于列名语义智能提取特征 - 废除 feature_start_column 硬编码位置索引 推理端 (inference_batch.py): - preprocess_spectra() 分支A: train_wavelengths存在→np.interp精确重采样 - preprocess_spectra() 分支B: 无波长元数据→解析列名→np.interp到400-800nm标准网格 - 废除暴力截断和零值填充
This commit is contained in:
@ -32,7 +32,7 @@ class Step8MlTrainHandler(BaseStepHandler):
|
||||
|
||||
try:
|
||||
result = ModelingStep.train_models(
|
||||
feature_start_column=config.get('feature_start_column', '374.285004'),
|
||||
feature_start_column=config.get('feature_start_column', None),
|
||||
preprocessing_methods=config.get('preprocessing_methods'),
|
||||
model_names=config.get('model_names'),
|
||||
split_methods=config.get('split_methods'),
|
||||
|
||||
@ -231,19 +231,125 @@ class WaterQualityModelingBatch:
|
||||
self.results = {}
|
||||
self.best_models = {}
|
||||
|
||||
def load_data_batch(self, csv_path: str, feature_start_column: Union[int, str]) -> Tuple[pd.DataFrame, Dict[str, pd.Series]]:
|
||||
"""
|
||||
批量加载CSV数据,将指定列之前的列作为目标值
|
||||
@staticmethod
|
||||
def _is_wavelength_column(col_name: str) -> bool:
|
||||
"""判断列名是否为波长值(纯数字字符串,如 '374.285004')"""
|
||||
try:
|
||||
float(str(col_name))
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
Args:
|
||||
csv_path: CSV文件路径
|
||||
feature_start_column: 特征开始列索引(int)或列名(str)
|
||||
@staticmethod
|
||||
def _is_wqi_column(col_name: str) -> bool:
|
||||
"""判断列名是否为 WQI 水质指数列('WQI_' 前缀)"""
|
||||
return str(col_name).startswith('WQI_')
|
||||
|
||||
@staticmethod
|
||||
def _extract_train_wavelengths(columns) -> List[float]:
|
||||
"""从列名列表中提取波长值(float 列表)
|
||||
|
||||
遍历列名,将所有可转为 float 的列名提取为波长列表。
|
||||
用于写入模型 metadata['train_wavelengths'],供推理端光谱重采样。
|
||||
"""
|
||||
wl_list = []
|
||||
for c in columns:
|
||||
try:
|
||||
wl_list.append(float(str(c)))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return wl_list
|
||||
|
||||
def _extract_feature_columns(self, data: pd.DataFrame,
|
||||
feature_start_column: Union[int, str, None] = None
|
||||
) -> Tuple[pd.DataFrame, List[int]]:
|
||||
"""从 DataFrame 中提取特征列(基于列名语义,而非位置索引)
|
||||
|
||||
策略(优先级从高到低):
|
||||
1) 如果 feature_start_column 是列名(str),定位该列并取之后的列
|
||||
2) 如果 feature_start_column 是整数,兼容旧逻辑(位置索引)
|
||||
3) 如果 feature_start_column 为 None,自动识别:
|
||||
- 保留所有纯数字列名(波长列)
|
||||
- 保留所有 WQI_ 前缀列(水质指数列)
|
||||
- 跳过坐标/元数据列
|
||||
|
||||
Returns:
|
||||
X: 特征数据
|
||||
X: 特征 DataFrame(保留列名)
|
||||
feature_col_indices: 特征列在原 data 中的位置索引列表
|
||||
"""
|
||||
all_cols = list(data.columns)
|
||||
|
||||
if feature_start_column is not None:
|
||||
# ── 兼容旧逻辑:按列名或索引位置截取 ──
|
||||
if isinstance(feature_start_column, str):
|
||||
if feature_start_column not in data.columns:
|
||||
raise ValueError(
|
||||
f"指定的特征开始列 '{feature_start_column}' 不存在于数据中"
|
||||
)
|
||||
start_idx = data.columns.get_loc(feature_start_column)
|
||||
print(f"[特征提取] 按列名 '{feature_start_column}' 定位 → 索引 {start_idx}")
|
||||
else:
|
||||
start_idx = int(feature_start_column)
|
||||
print(f"[特征提取] 按位置索引 {start_idx} 截取")
|
||||
|
||||
X = data.iloc[:, start_idx:]
|
||||
feature_indices = list(range(start_idx, len(all_cols)))
|
||||
else:
|
||||
# ── 智能识别:按列名语义过滤 ──
|
||||
# 黑名单:坐标列、元数据列(不会被误判为特征)
|
||||
_meta_patterns = {
|
||||
'x_coord', 'y_coord', 'pixel_x', 'pixel_y',
|
||||
'longitude', 'latitude', 'lon', 'lat',
|
||||
'id', 'station', 'sample_id',
|
||||
}
|
||||
feature_indices = []
|
||||
for i, col in enumerate(all_cols):
|
||||
col_lower = str(col).lower().strip()
|
||||
# 跳过元数据列
|
||||
if col_lower in _meta_patterns:
|
||||
continue
|
||||
# 保留波长列
|
||||
if self._is_wavelength_column(col):
|
||||
feature_indices.append(i)
|
||||
# 保留 WQI 列
|
||||
elif self._is_wqi_column(col):
|
||||
feature_indices.append(i)
|
||||
# 其他:跳过(可能是目标列或其他非特征列)
|
||||
|
||||
if not feature_indices:
|
||||
raise ValueError(
|
||||
"智能特征提取失败:未找到任何波长列或 WQI 列。"
|
||||
f"CSV 列名: {all_cols[:10]}..."
|
||||
)
|
||||
X = data.iloc[:, feature_indices]
|
||||
print(f"[特征提取] 智能识别: {len(feature_indices)} 个特征列 "
|
||||
f"(波长列 + WQI 列)")
|
||||
|
||||
print(f"[特征提取] 特征数据形状: {X.shape}")
|
||||
return X, feature_indices
|
||||
|
||||
def load_data_batch(self, csv_path: str,
|
||||
feature_start_column: Union[int, str, None] = None
|
||||
) -> Tuple[pd.DataFrame, Dict[str, pd.Series]]:
|
||||
"""批量加载 CSV 数据,自动识别特征列与目标列
|
||||
|
||||
改造要点(v2):
|
||||
- 特征列按列名语义提取(波长数字 / WQI_ 前缀),不再按硬编码位置一刀切
|
||||
- 目标列 = 不在特征列中、且非系统保留列(ID/坐标等)的数值列
|
||||
- X 保持为 DataFrame,列名保留波长信息(供后续提取 train_wavelengths)
|
||||
|
||||
Args:
|
||||
csv_path: CSV 文件路径
|
||||
feature_start_column: (可选)旧版兼容参数:
|
||||
- str: 特征起始列名
|
||||
- int: 特征起始列索引
|
||||
- None: 自动智能识别
|
||||
|
||||
Returns:
|
||||
X: 特征数据 (DataFrame,保留列名)
|
||||
y_dict: 目标值数据字典,键为列名
|
||||
"""
|
||||
# 读取CSV数据,处理空字符串和缺失值
|
||||
# 读取 CSV 数据,处理空字符串和缺失值
|
||||
try:
|
||||
data = pd.read_csv(csv_path, na_values=['', ' ', 'NaN', 'nan', 'NULL', 'null'])
|
||||
except pd.errors.EmptyDataError:
|
||||
@ -251,113 +357,105 @@ class WaterQualityModelingBatch:
|
||||
except Exception as e:
|
||||
raise ValueError(f"读取CSV文件 '{csv_path}' 时出错: {e}")
|
||||
|
||||
# 检查并清理数据中的空字符串和其他无效值
|
||||
print("数据清理...")
|
||||
original_shape = data.shape
|
||||
|
||||
# 将空字符串替换为NaN
|
||||
# 将空字符串替换为 NaN
|
||||
data = data.replace(r'^\s*$', np.nan, regex=True)
|
||||
|
||||
# 对于数值列,将无法转换为数字的字符串替换为NaN
|
||||
# 对于数值列,将无法转换为数字的字符串替换为 NaN
|
||||
for col in data.columns:
|
||||
try:
|
||||
# 尝试将列转换为数值类型
|
||||
data[col] = pd.to_numeric(data[col], errors='coerce')
|
||||
except Exception:
|
||||
# 如果转换失败,保持原样(可能是字符串列)
|
||||
pass
|
||||
|
||||
cleaned_shape = data.shape
|
||||
if cleaned_shape != original_shape:
|
||||
print(f"数据清理完成: {original_shape[0]}行{original_shape[1]}列 -> {cleaned_shape[0]}行{cleaned_shape[1]}列")
|
||||
|
||||
print(f"数据清理完成: {original_shape[0]}行{original_shape[1]}列 "
|
||||
f"-> {cleaned_shape[0]}行{cleaned_shape[1]}列")
|
||||
|
||||
print(f"数据加载完成,总列数: {data.shape[1]}")
|
||||
print(f"所有列名: {list(data.columns)}")
|
||||
|
||||
# 如果feature_start_column是列名,转换为索引
|
||||
if isinstance(feature_start_column, str):
|
||||
if feature_start_column not in data.columns:
|
||||
raise ValueError(f"指定的特征开始列 '{feature_start_column}' 不存在于数据中")
|
||||
feature_start_index = data.columns.get_loc(feature_start_column)
|
||||
print(f"特征开始列 '{feature_start_column}' 对应索引: {feature_start_index}")
|
||||
else:
|
||||
feature_start_index = feature_start_column
|
||||
print(f"特征开始列索引: {feature_start_index}")
|
||||
|
||||
# 提取特征数据(从feature_start_index开始)
|
||||
X = data.iloc[:, feature_start_index:]
|
||||
|
||||
# 提取所有目标列(从0列到feature_start_index-1列)
|
||||
print(f"所有列名: {list(data.columns)[:10]}..." if len(data.columns) > 10
|
||||
else f"所有列名: {list(data.columns)}")
|
||||
|
||||
# ── 提取特征列(基于列名语义) ──
|
||||
X, feature_indices = self._extract_feature_columns(
|
||||
data, feature_start_column
|
||||
)
|
||||
|
||||
# ── 提取目标列(不在特征列中 + 非系统保留列 + 数值类型) ──
|
||||
feature_idx_set = set(feature_indices)
|
||||
ignore_cols = {
|
||||
'ID', 'id', 'Id',
|
||||
'Longitude', 'Latitude', 'Lon', 'Lat',
|
||||
'longitude', 'latitude', 'lon', 'lat',
|
||||
'Station', 'station', 'sample_id',
|
||||
'x_coord', 'y_coord', 'pixel_x', 'pixel_y',
|
||||
}
|
||||
|
||||
y_dict = {}
|
||||
target_columns = data.columns[:feature_start_index]
|
||||
print(f"检测到的潜在目标列: {list(target_columns)}")
|
||||
|
||||
# 新增:跳过非预测目标的系统保留列
|
||||
ignore_cols = {'ID', 'id', 'Id', 'Longitude', 'Latitude', 'Lon', 'Lat', 'longitude', 'latitude', 'lon', 'lat', 'Station', 'station'}
|
||||
|
||||
for col_name in target_columns:
|
||||
# 过滤黑名单列
|
||||
if col_name in ignore_cols:
|
||||
print(f" 跳过目标列 '{col_name}': 属于系统保留列或空间坐标")
|
||||
for i, col_name in enumerate(data.columns):
|
||||
if i in feature_idx_set:
|
||||
continue # 已在特征矩阵中
|
||||
col_str = str(col_name).strip()
|
||||
if col_str in ignore_cols:
|
||||
print(f" 跳过 '{col_name}': 系统保留列")
|
||||
continue
|
||||
|
||||
y_series = data[col_name]
|
||||
|
||||
# 过滤非数值类型列 (避免将纯文本备注等拿去回归)
|
||||
if not pd.api.types.is_numeric_dtype(y_series):
|
||||
print(f" 跳过目标列 '{col_name}': 非数值类型")
|
||||
print(f" 跳过 '{col_name}': 非数值类型")
|
||||
continue
|
||||
if y_series.isna().all():
|
||||
print(f" 跳过 '{col_name}': 所有值为空")
|
||||
continue
|
||||
|
||||
# 检查是否有非空值
|
||||
if not y_series.isna().all():
|
||||
y_dict[col_name] = y_series
|
||||
print(f" 目标列 '{col_name}': {y_series.count()} 个非空值, 范围: {y_series.min():.4f} ~ {y_series.max():.4f}")
|
||||
else:
|
||||
print(f" 跳过目标列 '{col_name}': 所有值为空")
|
||||
|
||||
y_dict[col_name] = y_series
|
||||
print(f" 目标列 '{col_name}': {y_series.count()} 个非空值, "
|
||||
f"范围: {y_series.min():.4f} ~ {y_series.max():.4f}")
|
||||
|
||||
print(f"特征数据形状: {X.shape}")
|
||||
print(f"有效目标列数量: {len(y_dict)}")
|
||||
|
||||
|
||||
return X, y_dict
|
||||
|
||||
def load_data_single(self, csv_path: str, target_column_name: str, feature_start_column: Union[int, str]) -> Tuple[pd.DataFrame, pd.Series]:
|
||||
def load_data_single(self, csv_path: str, target_column_name: str,
|
||||
feature_start_column: Union[int, str, None] = None
|
||||
) -> Tuple[pd.DataFrame, pd.Series]:
|
||||
"""
|
||||
加载单个目标列的CSV数据
|
||||
加载单个目标列的CSV数据(v2:基于列名语义提取特征)
|
||||
|
||||
Args:
|
||||
csv_path: CSV文件路径
|
||||
target_column_name: 目标列名
|
||||
feature_start_column: 特征开始列索引(int)或列名(str)
|
||||
feature_start_column: 特征起始位置(可选,None=智能识别)
|
||||
|
||||
Returns:
|
||||
X: 特征数据
|
||||
X: 特征数据 (DataFrame,保留列名)
|
||||
y: 目标值数据
|
||||
"""
|
||||
data = pd.read_csv(csv_path)
|
||||
|
||||
|
||||
# 检查目标列是否存在
|
||||
if target_column_name not in data.columns:
|
||||
raise ValueError(f"目标列 '{target_column_name}' 不存在于数据中")
|
||||
|
||||
# 如果feature_start_column是列名,转换为索引
|
||||
if isinstance(feature_start_column, str):
|
||||
if feature_start_column not in data.columns:
|
||||
raise ValueError(f"指定的特征开始列 '{feature_start_column}' 不存在于数据中")
|
||||
feature_start_index = data.columns.get_loc(feature_start_column)
|
||||
else:
|
||||
feature_start_index = feature_start_column
|
||||
|
||||
# 提取目标值和特征
|
||||
|
||||
# 提取目标值
|
||||
y = data[target_column_name]
|
||||
X = data.iloc[:, feature_start_index:]
|
||||
|
||||
# 去除y值为空的行
|
||||
|
||||
# 去除 y 值为空的行
|
||||
mask = ~y.isna()
|
||||
data_cleaned = data[mask]
|
||||
|
||||
# 重新定义y和X,去除对应的空值行
|
||||
y = data_cleaned[target_column_name]
|
||||
X = data_cleaned.iloc[:, feature_start_column:]
|
||||
|
||||
# 提取特征(基于列名语义,排除目标列自身)
|
||||
X, _ = self._extract_feature_columns(
|
||||
data_cleaned, feature_start_column
|
||||
)
|
||||
# 确保目标列不在 X 中
|
||||
if target_column_name in X.columns:
|
||||
X = X.drop(columns=[target_column_name])
|
||||
|
||||
print(f"目标列 '{target_column_name}' 数据加载完成:")
|
||||
print(f" 样本数量: {X.shape[0]}")
|
||||
@ -744,6 +842,13 @@ class WaterQualityModelingBatch:
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
# 提取训练波长列表(写入 metadata,供推理端光谱重采样)
|
||||
train_wavelengths = self._extract_train_wavelengths(
|
||||
X_raw.columns
|
||||
)
|
||||
print(f"[波长记忆] 提取到 {len(train_wavelengths)} 个训练波长: "
|
||||
f"{train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm")
|
||||
|
||||
# 保存模型(result['model'] 已是 sklearn Pipeline)
|
||||
metadata = {
|
||||
'target_column_name': target_column_name,
|
||||
@ -761,9 +866,11 @@ class WaterQualityModelingBatch:
|
||||
'train_size': result['train_size'],
|
||||
'test_size': result['test_size'],
|
||||
'split_method': result['split_method'],
|
||||
# Pipeline 标记(便于旧 inference 路径兼容/诊断)
|
||||
'preprocess_method': preprocess_method,
|
||||
'is_pipeline': result.get('is_pipeline', False),
|
||||
# ★ 波长记忆:推理端可据此进行跨传感器光谱重采样
|
||||
'train_wavelengths': train_wavelengths,
|
||||
'train_columns': list(X_raw.columns),
|
||||
}
|
||||
|
||||
self.save_model(result['model'], target_column_name,
|
||||
|
||||
@ -377,110 +377,253 @@ class WaterQualityInference:
|
||||
|
||||
def preprocess_spectra(self, spectra: pd.DataFrame) -> np.ndarray:
|
||||
"""
|
||||
对光谱数据进行预处理
|
||||
对光谱数据进行预处理 + 跨传感器光谱重采样。
|
||||
|
||||
改造要点(v2 — 光谱重采样):
|
||||
- 废除原有的「列位置截断」和「零值填充」hack
|
||||
- 从 model metadata 中读取训练时的波长列表
|
||||
- 如果输入传感器与训练传感器波长网格不同,使用 np.interp()
|
||||
将输入光谱重采样到训练波长网格
|
||||
- 非光谱列(WQI 指数等)保持不变
|
||||
- 兼容旧模型(无 train_wavelengths 时回退到旧逻辑)
|
||||
|
||||
Args:
|
||||
spectra: 原始光谱数据
|
||||
spectra: 原始光谱数据(DataFrame,列名为波长字符串)
|
||||
|
||||
Returns:
|
||||
预处理后的光谱数据
|
||||
预处理后的光谱数据 (numpy ndarray)
|
||||
"""
|
||||
if self.loaded_model_data is None:
|
||||
raise ValueError("请先加载模型")
|
||||
|
||||
preprocess_method = self.loaded_model_data['preprocess_method']
|
||||
|
||||
# 处理 nan/NaN/None 值,转换为 "None" 字符串
|
||||
|
||||
# 处理 nan/NaN/None 值
|
||||
if pd.isna(preprocess_method) or str(preprocess_method).lower() in ['nan', 'none', '']:
|
||||
preprocess_method = "None"
|
||||
|
||||
|
||||
# 解析预处理方法(可能包含划分方法前缀)
|
||||
if '_' in str(preprocess_method):
|
||||
parts = str(preprocess_method).split('_')
|
||||
# 假设格式为 split_method_preprocess_method
|
||||
actual_preprocess_method = '_'.join(parts[1:]) if len(parts) > 1 else parts[-1]
|
||||
else:
|
||||
actual_preprocess_method = str(preprocess_method)
|
||||
|
||||
# 再次检查并转换 nan
|
||||
|
||||
if actual_preprocess_method.lower() in ['nan', 'none', '']:
|
||||
actual_preprocess_method = "None"
|
||||
|
||||
print(f"正在应用预处理方法: {actual_preprocess_method}")
|
||||
print(f"原始光谱数据形状: {spectra.shape}")
|
||||
|
||||
# ---- 自动特征补全:50 光谱 → 补全至模型训练时的 95 维(WQI 指数) ----
|
||||
# 触发条件:模型期望 n_features_in_ 个特征,但当前 spectra 列数不足
|
||||
# 原因:training_spectra.csv 含 50 光谱 + 45 WQI;sampling_spectra.csv 只有 50 光谱
|
||||
# 做法:与训练端(calculate_all_indices)完全一致的算法列表,实时补全缺失的 45 个 WQI 列
|
||||
model = self.loaded_model_data['model']
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
metadata = self.loaded_model_data.get('metadata', {})
|
||||
train_wavelengths = metadata.get('train_wavelengths', None)
|
||||
train_columns = metadata.get('train_columns', None)
|
||||
|
||||
# ---- 自动特征补全:50 光谱 → 补全至模型训练时的 n_features_in_ 维(WQI 指数) ----
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 < 模型期望 {expected_features} 列,"
|
||||
f"正在从光谱数据实时计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# ★ 核心:光谱重采样(跨传感器适配)
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
if train_wavelengths is not None and len(train_wavelengths) > 0:
|
||||
print(f"[光谱重采样] 模型训练波长: {len(train_wavelengths)} 个, "
|
||||
f"范围 {train_wavelengths[0]:.2f} ~ {train_wavelengths[-1]:.2f} nm")
|
||||
|
||||
# CSV 驱动的 WaterQualityIndexCalculator:所有公式名通过 list_available() 拿;
|
||||
# 一次性 calculate_many() 批量计算。彻底摆脱 dir(calc) 反射扫描 + 单 algo_func
|
||||
# 调用这种碎片化写法(Calculator 早已重构为公式驱动,不再有独立公式方法)。
|
||||
formulas = calc.list_available()
|
||||
if not formulas:
|
||||
print("[特征补全] Calculator 未持有任何公式,跳过补全")
|
||||
# 1) 从当前 spectra 中分离光谱列和 WQI 列
|
||||
target_wavelengths = []
|
||||
spectral_col_indices = []
|
||||
wqi_col_indices = []
|
||||
wqi_col_names = []
|
||||
|
||||
for i, col in enumerate(spectra.columns):
|
||||
col_str = str(col)
|
||||
try:
|
||||
wl = float(col_str)
|
||||
target_wavelengths.append(wl)
|
||||
spectral_col_indices.append(i)
|
||||
except (ValueError, TypeError):
|
||||
if col_str.startswith('WQI_'):
|
||||
wqi_col_indices.append(i)
|
||||
wqi_col_names.append(col_str)
|
||||
|
||||
print(f"[光谱重采样] 当前传感器: {len(target_wavelengths)} 个波长列, "
|
||||
f"{len(wqi_col_indices)} 个 WQI 列")
|
||||
|
||||
if len(target_wavelengths) == 0:
|
||||
print("[光谱重采样] ⚠ 未检测到波长列,跳过重采样")
|
||||
else:
|
||||
# 2) 提取光谱数据矩阵
|
||||
spectral_data = spectra.iloc[:, spectral_col_indices].values.astype(np.float64)
|
||||
|
||||
# 3) 逐行 np.interp 重采样
|
||||
train_wl_arr = np.array(train_wavelengths, dtype=np.float64)
|
||||
target_wl_arr = np.array(target_wavelengths, dtype=np.float64)
|
||||
|
||||
print(f"[光谱重采样] 执行重采样: {len(target_wavelengths)} → "
|
||||
f"{len(train_wavelengths)} 个波长点 ...")
|
||||
resampled = np.zeros((spectral_data.shape[0], len(train_wavelengths)),
|
||||
dtype=np.float64)
|
||||
for i in range(spectral_data.shape[0]):
|
||||
resampled[i] = np.interp(
|
||||
train_wl_arr, target_wl_arr, spectral_data[i],
|
||||
left=np.nan, right=np.nan
|
||||
)
|
||||
|
||||
# NaN 填充(外推区域):用最近有效值填充
|
||||
nan_mask = np.isnan(resampled)
|
||||
if nan_mask.any():
|
||||
print(f"[光谱重采样] {nan_mask.sum()} 个 NaN (外推区域) → 最近邻填充")
|
||||
for i in range(resampled.shape[0]):
|
||||
row_nan = np.isnan(resampled[i])
|
||||
if row_nan.any():
|
||||
valid = ~row_nan
|
||||
if valid.any():
|
||||
valid_idx = np.where(valid)[0]
|
||||
resampled[i, row_nan] = np.interp(
|
||||
np.where(row_nan)[0], valid_idx,
|
||||
resampled[i, valid]
|
||||
)
|
||||
else:
|
||||
resampled[i] = 0.0
|
||||
|
||||
# 4) 重组为 DataFrame(列名 = 训练波长字符串)
|
||||
wl_col_names = [f"{wl:.6f}" for wl in train_wavelengths]
|
||||
resampled_df = pd.DataFrame(resampled, columns=wl_col_names,
|
||||
index=spectra.index)
|
||||
|
||||
# 5) 拼接 WQI 列(如有)
|
||||
if wqi_col_indices:
|
||||
wqi_df = spectra.iloc[:, wqi_col_indices].copy()
|
||||
wqi_df.columns = wqi_col_names
|
||||
spectra = pd.concat([resampled_df, wqi_df], axis=1)
|
||||
else:
|
||||
# fast=True: Step9 特征补全走向量化快车道(~63 公式,已验证稳定)
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
# results_df 是列对齐的 WQI 计算结果(每列一个公式,行数=样本数)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成!光谱列已扩充至 {spectra.shape[1]} 列"
|
||||
f"(追加了 {spectra.shape[1] - original_col_count} 个 WQI 指数)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败,将使用原始光谱特征: {e}")
|
||||
spectra = resampled_df
|
||||
|
||||
# ---- 防线 1:强制维度对齐(物理截断)----
|
||||
if expected_features is not None and spectra.shape[1] > expected_features:
|
||||
print(f"[精准对齐] 正在将 {spectra.shape[1]} 维特征截断为模型要求的 {expected_features} 维")
|
||||
spectra = spectra.iloc[:, :expected_features]
|
||||
elif expected_features is not None and spectra.shape[1] < expected_features:
|
||||
# 维度不足时填充 0
|
||||
padding_cols = expected_features - spectra.shape[1]
|
||||
for i in range(padding_cols):
|
||||
spectra[f'_padding_{i}'] = 0.0
|
||||
print(f"[精准对齐] 特征不足,填充 {padding_cols} 列 0")
|
||||
print(f"[光谱重采样] 完成: {spectra.shape[1]} 列 "
|
||||
f"(光谱 {len(train_wavelengths)} + WQI {len(wqi_col_indices)})")
|
||||
|
||||
# ---- 防线 2:彻底清洗无穷大数值----
|
||||
# 防止 WQI 计算中除零/溢出产生 np.inf / -np.inf 导致预处理崩溃
|
||||
else:
|
||||
# ── 兼容旧模型(无 train_wavelengths)──
|
||||
print("[光谱重采样] 模型无 train_wavelengths 元数据(旧模型),"
|
||||
"跳过光谱重采样,进入兼容路径...")
|
||||
# 旧兼容逻辑:WQI 自动补全
|
||||
expected_features = getattr(model, 'n_features_in_', None)
|
||||
if expected_features is not None and spectra.shape[1] < expected_features:
|
||||
print(f"[特征补全] 检测到特征缺口:当前 {spectra.shape[1]} 列 "
|
||||
f"< 模型期望 {expected_features} 列,正在计算 WQI 指数...")
|
||||
try:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
formulas = calc.list_available()
|
||||
if formulas:
|
||||
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成!扩充至 {spectra.shape[1]} 列 "
|
||||
f"(+{spectra.shape[1] - original_col_count} WQI)")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败: {e}")
|
||||
|
||||
# ★ v4: 无 train_wavelengths 时,基于物理波长的经验重采样
|
||||
# 从 spectra 列名中解析真实波长 → np.interp 到 400-800nm 标准网格
|
||||
# 取代原基于列索引的硬截断(不同传感器列索引对应的波长完全不同)
|
||||
if expected_features is not None:
|
||||
n_current = spectra.shape[1]
|
||||
|
||||
# ── 第1步:从列名中提取波长列和非波长列 ──
|
||||
_target_wl: List[float] = []
|
||||
_wqi_names: List[str] = []
|
||||
_other_names: List[str] = []
|
||||
for _col in spectra.columns:
|
||||
try:
|
||||
_target_wl.append(float(str(_col)))
|
||||
except (ValueError, TypeError):
|
||||
if str(_col).startswith('WQI_'):
|
||||
_wqi_names.append(str(_col))
|
||||
else:
|
||||
_other_names.append(str(_col))
|
||||
|
||||
# 分离出 WQI 列(保存以便后续拼接回去)
|
||||
_wqi_df = spectra[_wqi_names].copy() if _wqi_names else pd.DataFrame()
|
||||
|
||||
if len(_target_wl) >= 2 and expected_features > 0:
|
||||
# ── 第2步:有真实波长 → np.interp 重采样 ──
|
||||
fallback_wl = np.linspace(400.0, 800.0, expected_features,
|
||||
dtype=np.float64)
|
||||
_target_arr = np.array(_target_wl, dtype=np.float64)
|
||||
_spec_data = spectra.iloc[:, :len(_target_wl)].values.astype(np.float64)
|
||||
# 按波长排序(以防列顺序不是严格递增)
|
||||
_sort_idx = np.argsort(_target_arr)
|
||||
_target_arr = _target_arr[_sort_idx]
|
||||
_spec_data = _spec_data[:, _sort_idx]
|
||||
|
||||
_resampled = np.zeros((_spec_data.shape[0], expected_features),
|
||||
dtype=np.float64)
|
||||
for i in range(_spec_data.shape[0]):
|
||||
_resampled[i] = np.interp(
|
||||
fallback_wl, _target_arr, _spec_data[i],
|
||||
left=np.nan, right=np.nan
|
||||
)
|
||||
# NaN 填充
|
||||
_nan = np.isnan(_resampled)
|
||||
if _nan.any():
|
||||
for i in range(_resampled.shape[0]):
|
||||
row_nan = np.isnan(_resampled[i])
|
||||
if row_nan.any():
|
||||
valid = ~row_nan
|
||||
if valid.any():
|
||||
vi = np.where(valid)[0]
|
||||
_resampled[i, row_nan] = np.interp(
|
||||
np.where(row_nan)[0], vi, _resampled[i, valid]
|
||||
)
|
||||
else:
|
||||
_resampled[i] = 0.0
|
||||
|
||||
# 重组 DataFrame
|
||||
_wl_cols = [f"{w:.1f}" for w in fallback_wl]
|
||||
spectra = pd.DataFrame(_resampled, columns=_wl_cols,
|
||||
index=spectra.index)
|
||||
if not _wqi_df.empty:
|
||||
spectra = pd.concat([spectra, _wqi_df.reset_index(drop=True)],
|
||||
axis=1)
|
||||
print(f"[兼容重采样] 列名解析到 {len(_target_wl)} 个波长 "
|
||||
f"({_target_arr[0]:.1f}~{_target_arr[-1]:.1f}nm) → "
|
||||
f"np.interp → {expected_features} 列 (400~800nm 标准网格)"
|
||||
f"{' + ' + str(len(_wqi_names)) + ' WQI' if _wqi_names else ''}")
|
||||
|
||||
elif n_current > expected_features:
|
||||
# ── 第3步:无法解析波长(纯字符串列名)→ 硬截断兜底 ──
|
||||
spectra = spectra.iloc[:, :expected_features]
|
||||
print(f"[兼容截断] 无法解析波长列名,硬截断 "
|
||||
f"{n_current} → {expected_features} 列")
|
||||
|
||||
elif n_current < expected_features:
|
||||
# ── 第4步:特征不够 → 补零 ──
|
||||
for i in range(expected_features - n_current):
|
||||
spectra[f'_padding_{i}'] = 0.0
|
||||
print(f"[兼容填充] 特征不足,补零 "
|
||||
f"{n_current} → {expected_features} 列")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 通用清洗
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
spectra = spectra.replace([np.inf, -np.inf], np.nan)
|
||||
spectra = spectra.fillna(0)
|
||||
|
||||
print(f"[特征对齐] 最终输入维度: {spectra.shape}")
|
||||
|
||||
# ---- Pipeline 化分支:模型内置 scaler/MSC.mean_spectrum_ 等状态时,跳过手动 Preprocessing ----
|
||||
# ── Pipeline 化分支:模型内置 scaler → 跳过手动 Preprocessing ──
|
||||
if isinstance(model, Pipeline):
|
||||
print(f"[Pipeline] 检测到模型是 sklearn Pipeline,"
|
||||
f"其内置预处理步骤({list(model.named_steps.keys())[0]})将处理原始光谱,"
|
||||
print(f"[Pipeline] 模型是 sklearn Pipeline,内置预处理步骤,"
|
||||
f"无需外部 Preprocessing")
|
||||
return spectra.values
|
||||
|
||||
# ---- 兼容路径:旧 .joblib(裸模型 + preprocess_method 字符串)回退手动 Preprocessing ----
|
||||
# ── 兼容路径:旧裸模型 + 手动 Preprocessing ──
|
||||
try:
|
||||
# 应用预处理
|
||||
spectra_processed = Preprocessing(actual_preprocess_method, spectra)
|
||||
|
||||
# 确保返回numpy数组
|
||||
if isinstance(spectra_processed, pd.DataFrame):
|
||||
spectra_processed = spectra_processed.values
|
||||
|
||||
print(f" [Legacy] 旧裸模型 + 手动 Preprocessing({actual_preprocess_method}) 完成,"
|
||||
f"数据形状: {spectra_processed.shape}")
|
||||
|
||||
return spectra_processed
|
||||
|
||||
except Exception as e:
|
||||
print(f"预处理失败: {e}")
|
||||
print("使用原始数据")
|
||||
|
||||
@ -129,7 +129,7 @@ class ModelingStep:
|
||||
|
||||
@staticmethod
|
||||
def train_models(
|
||||
feature_start_column: str = "374.285004",
|
||||
feature_start_column: Union[int, str, None] = None,
|
||||
preprocessing_methods: Optional[List[str]] = None,
|
||||
model_names: Optional[List[str]] = None,
|
||||
split_methods: Optional[List[str]] = None,
|
||||
|
||||
@ -528,16 +528,14 @@ class Step8MlTrainPanel(QWidget):
|
||||
|
||||
def get_training_params(self):
|
||||
"""获取模型训练参数"""
|
||||
# ★ 2026-07-01 安全保护:float() 前验证 currentText() 是否为有效数值
|
||||
# ★ v2:currentText() 为列名字符串,传 None 走智能识别
|
||||
feature_text = self.feature_start.currentText()
|
||||
try:
|
||||
feature_start = float(feature_text)
|
||||
except (ValueError, TypeError):
|
||||
feature_start = 374.285004 # 默认光谱起始列
|
||||
if not feature_text or feature_text.startswith("("):
|
||||
feature_text = None
|
||||
|
||||
return {
|
||||
'pipeline_type': 'machine_learning',
|
||||
'feature_start': feature_start,
|
||||
'feature_start': feature_text,
|
||||
'cv_folds': self.cv_folds.value(),
|
||||
'preprocess_methods': [method for method, cb in self.preproc_checkboxes.items() if cb.isChecked()],
|
||||
'model_types': [model for model, cb in self.model_checkboxes.items() if cb.isChecked()],
|
||||
|
||||
@ -14,7 +14,7 @@ Step8 后端计算服务(机器学习建模训练)
|
||||
|
||||
execute_step8({
|
||||
"training_csv_path": "D:/training_spectra_indices.csv", # 训练 CSV(必填)
|
||||
"feature_start_column": "374.285004", # 特征起始列名/索引
|
||||
"feature_start_column": None, # None=智能识别特征列
|
||||
"preprocessing_methods": ["None", "MMS"], # 预处理方法列表
|
||||
"model_names": ["RF", "SVR", "Ridge", "Lasso"], # 模型列表
|
||||
"split_methods": ["spxy"], # 划分方法列表
|
||||
@ -68,7 +68,7 @@ def execute_step8(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
# ---------- 入参规整 ----------
|
||||
training_csv_path: Optional[str] = config.get("training_csv_path")
|
||||
feature_start_column: str = str(config.get("feature_start_column", "374.285004"))
|
||||
feature_start_column: Optional[str] = config.get("feature_start_column", None)
|
||||
preprocessing_methods: Optional[List[str]] = config.get("preprocessing_methods")
|
||||
model_names: Optional[List[str]] = config.get("model_names")
|
||||
split_methods: Optional[List[str]] = config.get("split_methods")
|
||||
|
||||
@ -104,12 +104,12 @@ class Step8View(BaseView):
|
||||
params_layout = QFormLayout()
|
||||
|
||||
self.feature_start = QLineEdit()
|
||||
self.feature_start.setText("374.285004")
|
||||
self.feature_start.setPlaceholderText("留空=智能识别波长列和WQI列")
|
||||
params_layout.addRow("特征起始列:", self.feature_start)
|
||||
|
||||
feature_start_hint = QLabel(
|
||||
"提示:请使用记事本打开 training_spectra.csv 确认首个波长的精确表头名称"
|
||||
"(如 374.285 或 374.285004)并在此填入,避免因浮点精度差异导致列名匹配失败。"
|
||||
"提示:留空则自动识别所有波长列(纯数字列名)和 WQI 指数列作为特征。\n"
|
||||
"如需手动指定,可填入首个波长列名(如 374.285004)或列索引。"
|
||||
)
|
||||
feature_start_hint.setWordWrap(True)
|
||||
feature_start_hint.setStyleSheet("color: #666; font-size: 10px;")
|
||||
@ -253,7 +253,7 @@ class Step8View(BaseView):
|
||||
]
|
||||
|
||||
config = {
|
||||
"feature_start_column": self.feature_start.text(),
|
||||
"feature_start_column": self.feature_start.text().strip() or None,
|
||||
"preprocessing_methods": preprocessing_methods if preprocessing_methods else ["None"],
|
||||
"model_names": model_names if model_names else ["SVR"],
|
||||
"split_methods": split_methods if split_methods else ["random"],
|
||||
@ -270,7 +270,8 @@ class Step8View(BaseView):
|
||||
|
||||
def set_config(self, config: dict):
|
||||
if "feature_start_column" in config:
|
||||
self.feature_start.setText(str(config["feature_start_column"]))
|
||||
val = config["feature_start_column"]
|
||||
self.feature_start.setText(str(val) if val is not None else "")
|
||||
if "cv_folds" in config:
|
||||
self.cv_folds.setValue(config["cv_folds"])
|
||||
if "preprocessing_methods" in config:
|
||||
|
||||
Reference in New Issue
Block a user