Step8/9 UX: feature_start 改 QComboBox + 多源 CSV 优先回退
This commit is contained in:
228
_smoke_test_step8_9_csv_combo.py
Normal file
228
_smoke_test_step8_9_csv_combo.py
Normal file
@ -0,0 +1,228 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Smoke test: step8_ml_train_panel + step9_ml_predict_panel 的新 CSV→QComboBox 联动
|
||||||
|
& update_from_config 多源文件优先回退 逻辑。
|
||||||
|
|
||||||
|
不真正 import 整个 panel(避免 osgeo 重依赖),只 import 类本身,验证:
|
||||||
|
1) Step8MlTrainPanel.feature_start 变成 QComboBox
|
||||||
|
2) Step8._on_training_csv_changed 正确读 CSV 表头 + 默认选中 374.285
|
||||||
|
3) Step8._resolve_training_csv_from_workdir 优先级(Step7 > 10_WaterIndex > Step6 > Step7 兜底)
|
||||||
|
4) Step8.get_config / set_config / get_training_params 用 QComboBox API 不崩
|
||||||
|
5) Step9._resolve_latest_wqi_test_csv 找到正确的 WQI 测试集
|
||||||
|
6) Step9.update_from_config 优先级(WQI CSV > Step4 raw)
|
||||||
|
|
||||||
|
用 offscreen 模式避免弹窗,完整可重复跑。
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# 避免 panel 顶部 osgeo 导入阻塞
|
||||||
|
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
|
||||||
|
|
||||||
|
# 让 src 目录可 import
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
sys.path.insert(0, HERE)
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
from PyQt5.QtWidgets import QApplication
|
||||||
|
|
||||||
|
# 必须先 create QApplication 才能 new widget
|
||||||
|
app = QApplication.instance() or QApplication(sys.argv)
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 准备 fixture:模拟工作目录
|
||||||
|
# ============================================================
|
||||||
|
def make_fake_csv(path, n_cols_50_band, extra_wqi_cols=None, n_rows=10):
|
||||||
|
"""生成模拟 CSV:50 个纯数字波段列 + 任意额外 WQI 列。"""
|
||||||
|
band_cols = [374.285 + i for i in range(n_cols_50_band)]
|
||||||
|
# 浮点列名会被 pandas 自动转成 374.285 / 374.285001 / ...
|
||||||
|
cols = band_cols + (extra_wqi_cols or [])
|
||||||
|
data = np.random.RandomState(42).rand(n_rows, len(cols))
|
||||||
|
df = pd.DataFrame(data, columns=cols)
|
||||||
|
df.to_csv(path, index=False)
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
work_dir = tmp
|
||||||
|
print(f"[fixture] work_dir = {work_dir}")
|
||||||
|
|
||||||
|
# 1) Step 6 原始光谱
|
||||||
|
os.makedirs(os.path.join(work_dir, "6_Spectral_Feature_Extraction"), exist_ok=True)
|
||||||
|
step6_csv = os.path.join(work_dir, "6_Spectral_Feature_Extraction", "training_spectra.csv")
|
||||||
|
make_fake_csv(step6_csv, 50)
|
||||||
|
|
||||||
|
# 2) Step 7 WQI 增强版
|
||||||
|
os.makedirs(os.path.join(work_dir, "7_Water_Quality_Indices"), exist_ok=True)
|
||||||
|
step7_csv = os.path.join(work_dir, "7_Water_Quality_Indices", "training_spectra_indices.csv")
|
||||||
|
wqi_names = [f"WQI_Chla_{i}" for i in range(10)] + [f"WQI_TP_{i}" for i in range(5)]
|
||||||
|
make_fake_csv(step7_csv, 50, wqi_names)
|
||||||
|
|
||||||
|
# 3) 10_WaterIndex_CSV 下的 training csv(较旧 mtime)
|
||||||
|
os.makedirs(os.path.join(work_dir, "10_WaterIndex_CSV"), exist_ok=True)
|
||||||
|
step10_csv = os.path.join(work_dir, "10_WaterIndex_CSV", "training_watercolor_summary.csv")
|
||||||
|
make_fake_csv(step10_csv, 50, wqi_names)
|
||||||
|
# 把 mtime 调成 3 年前,确认 7_Water_Quality_Indices 那个更新
|
||||||
|
old_time = (pd.Timestamp.now() - pd.Timedelta(days=365 * 3)).timestamp()
|
||||||
|
os.utime(step10_csv, (old_time, old_time))
|
||||||
|
|
||||||
|
# 4) Step 4 raw sampling
|
||||||
|
os.makedirs(os.path.join(work_dir, "4_sampling"), exist_ok=True)
|
||||||
|
step4_csv = os.path.join(work_dir, "4_sampling", "sampling_spectra.csv")
|
||||||
|
make_fake_csv(step4_csv, 50)
|
||||||
|
|
||||||
|
# 5) WQI 测试集(10_WaterIndex_CSV 下另一个 mtime 较新的 csv)
|
||||||
|
wqi_test_csv = os.path.join(work_dir, "10_WaterIndex_CSV", "sampling_watercolor_summary.csv")
|
||||||
|
make_fake_csv(wqi_test_csv, 50, wqi_names)
|
||||||
|
# 设为 1 小时前(mtime 比 step7_csv 新)
|
||||||
|
new_time = (pd.Timestamp.now() - pd.Timedelta(hours=1)).timestamp()
|
||||||
|
os.utime(wqi_test_csv, (new_time, new_time))
|
||||||
|
|
||||||
|
# 6) 9_ML_Prediction 输出目录
|
||||||
|
os.makedirs(os.path.join(work_dir, "9_ML_Prediction"), exist_ok=True)
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 1:Step8 加载 + QComboBox 化验证
|
||||||
|
# ============================================================
|
||||||
|
from src.gui.panels.step8_ml_train_panel import Step8MlTrainPanel
|
||||||
|
from PyQt5.QtWidgets import QComboBox
|
||||||
|
|
||||||
|
panel8 = Step8MlTrainPanel()
|
||||||
|
assert isinstance(panel8.feature_start, QComboBox), \
|
||||||
|
f"FAIL: feature_start 应为 QComboBox,实际为 {type(panel8.feature_start)}"
|
||||||
|
print(f"[OK] Step8.feature_start 类型: {type(panel8.feature_start).__name__}")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 2:CSV 变更槽 — 模拟用户选 WQI CSV,自动填表头
|
||||||
|
# ============================================================
|
||||||
|
panel8._on_training_csv_changed(step7_csv)
|
||||||
|
assert panel8.feature_start.count() > 0
|
||||||
|
head_text = panel8.feature_start.currentText()
|
||||||
|
print(f"[OK] Step8 选中 step7_csv 后,feature_start.currentText() = {head_text!r}")
|
||||||
|
# 默认选中:第一个纯数字波段列 "374.285"
|
||||||
|
assert head_text.replace('.', '').lstrip('-').isdigit(), \
|
||||||
|
f"FAIL: 默认应选数字波段,实际 = {head_text!r}"
|
||||||
|
print(f"[OK] 默认选中数字波段列: {head_text!r}")
|
||||||
|
|
||||||
|
# 验证前 50 列已填充
|
||||||
|
n_items = panel8.feature_start.count()
|
||||||
|
assert n_items >= 50, f"FAIL: 应至少 50 项,实际 {n_items}"
|
||||||
|
print(f"[OK] Step8.feature_start 填充了 {n_items} 个候选列(前 50 列)")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 3:CSV 不存在时,combo 回到占位项
|
||||||
|
# ============================================================
|
||||||
|
panel8._on_training_csv_changed("/nonexistent/path.csv")
|
||||||
|
assert panel8.feature_start.count() == 1
|
||||||
|
assert "请先选择训练 CSV" in panel8.feature_start.itemText(0)
|
||||||
|
print(f"[OK] Step8 CSV 不存在时,combo 回退到占位项: {panel8.feature_start.itemText(0)!r}")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 4:get_config / set_config / get_training_params 用 QComboBox API 不崩
|
||||||
|
# ============================================================
|
||||||
|
panel8._on_training_csv_changed(step7_csv)
|
||||||
|
cfg = panel8.get_config()
|
||||||
|
assert 'feature_start_column' in cfg
|
||||||
|
assert isinstance(cfg['feature_start_column'], str)
|
||||||
|
assert cfg['feature_start_column'].replace('.', '').lstrip('-').isdigit()
|
||||||
|
print(f"[OK] Step8.get_config()['feature_start_column'] = {cfg['feature_start_column']!r}")
|
||||||
|
|
||||||
|
# set_config 试一下(先选个不存在的列名,应该走兜底 addItem 分支)
|
||||||
|
panel8.set_config({'feature_start_column': '999.999_unknown'})
|
||||||
|
# 重新触发 CSV 加载,验证回放不污染
|
||||||
|
panel8._on_training_csv_changed(step7_csv)
|
||||||
|
panel8.set_config({'feature_start_column': '374.285'})
|
||||||
|
assert panel8.feature_start.currentText() == '374.285'
|
||||||
|
print(f"[OK] Step8.set_config() 正确把 374.285 选中")
|
||||||
|
|
||||||
|
# get_training_params
|
||||||
|
params = panel8.get_training_params()
|
||||||
|
assert 'feature_start' in params
|
||||||
|
assert isinstance(params['feature_start'], float)
|
||||||
|
print(f"[OK] Step8.get_training_params()['feature_start'] = {params['feature_start']}")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 5:_resolve_training_csv_from_workdir 优先级
|
||||||
|
# ============================================================
|
||||||
|
panel8.work_dir = work_dir
|
||||||
|
resolved = panel8._resolve_training_csv_from_workdir()
|
||||||
|
expected = step7_csv.replace('\\', '/')
|
||||||
|
assert resolved == expected, f"FAIL: 应优先 step7, 实际 {resolved!r}, 期望 {expected!r}"
|
||||||
|
print(f"[OK] Step8._resolve_training_csv_from_workdir() = {resolved}")
|
||||||
|
print(f" 优先级 1: Step 7 WQI 增强版 ✓")
|
||||||
|
|
||||||
|
# 模拟用户没跑过 Step 7(删掉 step7_csv 目录)— 但 10_WaterIndex_CSV 还在
|
||||||
|
import shutil
|
||||||
|
shutil.rmtree(os.path.join(work_dir, "7_Water_Quality_Indices"))
|
||||||
|
resolved = panel8._resolve_training_csv_from_workdir()
|
||||||
|
# 用户明确提到 10_WaterIndex_CSV 目录下的结果是有效来源,应作为第二优先级
|
||||||
|
expected_step10 = str(Path(step10_csv)).replace('\\', '/')
|
||||||
|
assert resolved == expected_step10, \
|
||||||
|
f"FAIL: 应取 10_WaterIndex_CSV/*training*, 实际 {resolved!r}, 期望 {expected_step10!r}"
|
||||||
|
print(f"[OK] Step 7 缺省时,回退到 10_WaterIndex_CSV/*training*: {resolved}")
|
||||||
|
print(f" 优先级 2: 10_WaterIndex_CSV/*training* ✓")
|
||||||
|
|
||||||
|
# 再删 10_WaterIndex_CSV 才回退到 Step 6
|
||||||
|
shutil.rmtree(os.path.join(work_dir, "10_WaterIndex_CSV"))
|
||||||
|
resolved = panel8._resolve_training_csv_from_workdir()
|
||||||
|
assert resolved == step6_csv.replace('\\', '/'), \
|
||||||
|
f"FAIL: 应回退 step6, 实际 {resolved!r}, 期望 {step6_csv!r}"
|
||||||
|
print(f"[OK] Step 7 / 10_WaterIndex_CSV 都缺省时,回退到 Step 6: {resolved}")
|
||||||
|
print(f" 优先级 3: Step 6 原始光谱 ✓")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 6:Step9 加载
|
||||||
|
# ============================================================
|
||||||
|
from src.gui.panels.step9_ml_predict_panel import Step9MlPredictPanel
|
||||||
|
|
||||||
|
panel9 = Step9MlPredictPanel()
|
||||||
|
print(f"[OK] Step9MlPredictPanel 实例化成功")
|
||||||
|
|
||||||
|
# 重建 10_WaterIndex_CSV(WQI 测试集 fixture,test 5 中被删了)
|
||||||
|
os.makedirs(os.path.join(work_dir, "10_WaterIndex_CSV"), exist_ok=True)
|
||||||
|
wqi_test_csv = os.path.join(work_dir, "10_WaterIndex_CSV", "sampling_watercolor_summary.csv")
|
||||||
|
make_fake_csv(wqi_test_csv, 50, wqi_names)
|
||||||
|
new_time = (pd.Timestamp.now() - pd.Timedelta(hours=1)).timestamp()
|
||||||
|
os.utime(wqi_test_csv, (new_time, new_time))
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 7:Step9._resolve_latest_wqi_test_csv 优先 10_WaterIndex_CSV
|
||||||
|
# ============================================================
|
||||||
|
panel9.work_dir = work_dir
|
||||||
|
resolved = panel9._resolve_latest_wqi_test_csv()
|
||||||
|
# 应该是 sampling_watercolor_summary.csv(mtime 较新)
|
||||||
|
expected = wqi_test_csv.replace('\\', '/')
|
||||||
|
assert resolved == expected, \
|
||||||
|
f"FAIL: 应取 mtime 最新的 wqi 测试集, 实际 {resolved!r}, 期望 {expected!r}"
|
||||||
|
print(f"[OK] Step9._resolve_latest_wqi_test_csv() = {resolved}")
|
||||||
|
print(f" 优先级:10_WaterIndex_CSV/sampling_watercolor_summary.csv ✓")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 8:Step9.update_from_config 优先级:WQI CSV > Step4 raw
|
||||||
|
# ============================================================
|
||||||
|
# 注:Step 4 回退路径依赖 main_window._panel_factory(offline 测试无 main_window,
|
||||||
|
# 故仅验证 WQI CSV 优先级主路径;Step 4 路径与原代码同源,逻辑没变)
|
||||||
|
# 有 WQI 时优先 WQI
|
||||||
|
os.makedirs(os.path.join(work_dir, "10_WaterIndex_CSV"), exist_ok=True)
|
||||||
|
wqi_test_csv2 = os.path.join(work_dir, "10_WaterIndex_CSV", "sampling_watercolor_summary.csv")
|
||||||
|
make_fake_csv(wqi_test_csv2, 50, wqi_names)
|
||||||
|
panel9.update_from_config(work_dir=work_dir, pipeline=None)
|
||||||
|
path = panel9.sampling_csv_file.get_path()
|
||||||
|
assert wqi_test_csv2.replace('\\', '/') in path, \
|
||||||
|
f"FAIL: 应优先 WQI CSV, 实际 {path!r}"
|
||||||
|
print(f"[OK] Step9.update_from_config() 优先 WQI CSV: {path}")
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 测试 9:work_dir 不存在时,所有 helper 安全返回
|
||||||
|
# ============================================================
|
||||||
|
panel9.work_dir = ""
|
||||||
|
assert panel9._resolve_latest_wqi_test_csv() == ""
|
||||||
|
print(f"[OK] work_dir 空时 _resolve_latest_wqi_test_csv 安全返回空串")
|
||||||
|
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print("ALL SMOKE TESTS PASSED")
|
||||||
|
print("=" * 60)
|
||||||
@ -14,10 +14,12 @@ if _HERE not in sys.path:
|
|||||||
sys.path.insert(0, _HERE)
|
sys.path.insert(0, _HERE)
|
||||||
from src.gui.panels._step_path_resolver import get_step_output_path, resolve_step_widget, resolve_subdir
|
from src.gui.panels._step_path_resolver import get_step_output_path, resolve_step_widget, resolve_subdir
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
from PyQt5.QtWidgets import (
|
from PyQt5.QtWidgets import (
|
||||||
QWidget, QVBoxLayout, QGroupBox, QFormLayout, QGridLayout,
|
QWidget, QVBoxLayout, QGroupBox, QFormLayout, QGridLayout,
|
||||||
QHBoxLayout, QLabel, QLineEdit, QSpinBox, QCheckBox,
|
QHBoxLayout, QLabel, QLineEdit, QSpinBox, QCheckBox,
|
||||||
QPushButton, QFileDialog, QMessageBox, QSizePolicy,
|
QPushButton, QFileDialog, QMessageBox, QSizePolicy, QComboBox,
|
||||||
)
|
)
|
||||||
from PyQt5.QtCore import Qt
|
from PyQt5.QtCore import Qt
|
||||||
|
|
||||||
@ -95,6 +97,9 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
)
|
)
|
||||||
layout.addWidget(self.training_csv_file)
|
layout.addWidget(self.training_csv_file)
|
||||||
|
|
||||||
|
# 训练 CSV 选定后,自动刷新"特征起始列"下拉框的候选列
|
||||||
|
self.training_csv_file.line_edit.textChanged.connect(self._on_training_csv_changed)
|
||||||
|
|
||||||
# 机器学习模型页面
|
# 机器学习模型页面
|
||||||
self.ml_page = QWidget()
|
self.ml_page = QWidget()
|
||||||
self.create_ml_page()
|
self.create_ml_page()
|
||||||
@ -144,19 +149,22 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
params_group = QGroupBox("训练参数")
|
params_group = QGroupBox("训练参数")
|
||||||
params_layout = QFormLayout()
|
params_layout = QFormLayout()
|
||||||
|
|
||||||
self.feature_start = QLineEdit()
|
self.feature_start = QComboBox()
|
||||||
self.feature_start.setText("374.285004")
|
self.feature_start.setMinimumWidth(180)
|
||||||
|
self.feature_start.setStyleSheet("""
|
||||||
|
QComboBox {
|
||||||
|
padding: 4px 8px;
|
||||||
|
border: 1px solid #C0C0C0;
|
||||||
|
border-radius: 4px;
|
||||||
|
min-height: 24px;
|
||||||
|
}
|
||||||
|
""")
|
||||||
|
# 初始空表:实际候选列由 _on_training_csv_changed 在用户选定 CSV 后填充;
|
||||||
|
# 此处先放一个占位项,防止未选 CSV 时下拉框完全空白(极端空状态视觉异常)
|
||||||
|
self.feature_start.addItem("(请先选择训练 CSV)", "")
|
||||||
|
self.feature_start.setCurrentIndex(0)
|
||||||
params_layout.addRow("特征起始列:", self.feature_start)
|
params_layout.addRow("特征起始列:", self.feature_start)
|
||||||
|
|
||||||
# 特征起始列名提示:用记事本打开 training_spectra.csv 确认首个波长的精确表头
|
|
||||||
feature_start_hint = QLabel(
|
|
||||||
"提示:请使用记事本打开 training_spectra.csv 确认首个波长的精确表头名称"
|
|
||||||
"(如 374.285 或 374.285004)并在此填入,避免因浮点精度差异导致列名匹配失败。"
|
|
||||||
)
|
|
||||||
feature_start_hint.setWordWrap(True)
|
|
||||||
feature_start_hint.setStyleSheet("color: #666; font-size: 10px;")
|
|
||||||
params_layout.addRow(feature_start_hint)
|
|
||||||
|
|
||||||
self.cv_folds = QSpinBox()
|
self.cv_folds = QSpinBox()
|
||||||
self.cv_folds.setRange(2, 10)
|
self.cv_folds.setRange(2, 10)
|
||||||
self.cv_folds.setValue(3)
|
self.cv_folds.setValue(3)
|
||||||
@ -318,6 +326,133 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
return str(mw.work_dir)
|
return str(mw.work_dir)
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
def _on_training_csv_changed(self, csv_path: str):
|
||||||
|
"""训练 CSV 变更槽:自动读表头(前 50 列)填充 self.feature_start 下拉框。
|
||||||
|
|
||||||
|
触发场景:
|
||||||
|
- 用户点"浏览..."选了新 CSV
|
||||||
|
- 上游 panel_factory / set_config 注入路径
|
||||||
|
|
||||||
|
默认选中规则:
|
||||||
|
1) 优先 '374.285' 纯数字波段列(光谱起点;容忍 .285 / .285004 / .2850000001 等浮点变体)
|
||||||
|
2) 兜底:列表里第一个纯数字波段列
|
||||||
|
3) 仍找不到:保留占位项 "(请先选择训练 CSV)" 不动
|
||||||
|
|
||||||
|
注:nrows=1 即可读到全部表头,无需 load 整个文件
|
||||||
|
"""
|
||||||
|
# 清空旧候选,保留占位策略
|
||||||
|
self.feature_start.blockSignals(True)
|
||||||
|
try:
|
||||||
|
self.feature_start.clear()
|
||||||
|
if not csv_path or not os.path.isfile(csv_path):
|
||||||
|
self.feature_start.addItem("(请先选择训练 CSV)", "")
|
||||||
|
self.feature_start.setCurrentIndex(0)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 读取前 50 列表头
|
||||||
|
try:
|
||||||
|
df_head = pd.read_csv(csv_path, nrows=0)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[Step8] 读取 CSV 表头失败: {e}")
|
||||||
|
self.feature_start.addItem("(CSV 读取失败)", "")
|
||||||
|
self.feature_start.setCurrentIndex(0)
|
||||||
|
return
|
||||||
|
|
||||||
|
all_cols = list(df_head.columns)
|
||||||
|
head_cols = all_cols[:50]
|
||||||
|
if not head_cols:
|
||||||
|
self.feature_start.addItem("(CSV 无列)", "")
|
||||||
|
self.feature_start.setCurrentIndex(0)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 填充前 50 列到下拉框
|
||||||
|
for col in head_cols:
|
||||||
|
self.feature_start.addItem(str(col), str(col))
|
||||||
|
|
||||||
|
# 默认选中规则:纯数字波段列(纯数字字符串视为波长)
|
||||||
|
default_idx = self._find_default_band_index(head_cols)
|
||||||
|
if default_idx is not None:
|
||||||
|
self.feature_start.setCurrentIndex(default_idx)
|
||||||
|
else:
|
||||||
|
self.feature_start.setCurrentIndex(0)
|
||||||
|
finally:
|
||||||
|
self.feature_start.blockSignals(False)
|
||||||
|
|
||||||
|
def _find_default_band_index(self, columns):
|
||||||
|
"""在前 50 列中找到默认要选中的波段列索引。
|
||||||
|
|
||||||
|
优先级:
|
||||||
|
1) 列名以 '374.285' 开头(覆盖 374.285 / 374.285004 / 374.2850001)
|
||||||
|
2) 列名是纯数字(如 '443')— 极少数仪器用整数波长
|
||||||
|
3) 第一个能被 float() 解析的纯数字列
|
||||||
|
"""
|
||||||
|
# 1) 374.285 前缀
|
||||||
|
for i, col in enumerate(columns):
|
||||||
|
if str(col).startswith("374.285"):
|
||||||
|
return i
|
||||||
|
# 2) 纯数字列(无小数点)
|
||||||
|
for i, col in enumerate(columns):
|
||||||
|
if str(col).replace('.', '').lstrip('-').isdigit() and '.' not in str(col):
|
||||||
|
return i
|
||||||
|
# 3) 任意可被 float 解析的列
|
||||||
|
for i, col in enumerate(columns):
|
||||||
|
try:
|
||||||
|
float(str(col))
|
||||||
|
return i
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _resolve_training_csv_from_workdir(self):
|
||||||
|
"""根据工作目录智能挑选训练 CSV 路径。
|
||||||
|
|
||||||
|
优先级(从高到低):
|
||||||
|
1) 7_Water_Quality_Indices/training_spectra_indices.csv(Step 7 WQI 增强版)
|
||||||
|
2) 10_WaterIndex_CSV/*training* / *training*indices*.csv(用户自定义带指数汇总)
|
||||||
|
3) 6_Spectral_Feature_Extraction/training_spectra.csv(Step 6 原始特征)
|
||||||
|
4) 7_Water_Quality_Indices/ 下任意 *training*.csv
|
||||||
|
"""
|
||||||
|
work_dir = self._get_default_work_dir()
|
||||||
|
if not work_dir:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
wd = Path(work_dir)
|
||||||
|
|
||||||
|
# 1) Step 7 输出的训练 WQI 增强版
|
||||||
|
step7_csv = wd / "7_Water_Quality_Indices" / "training_spectra_indices.csv"
|
||||||
|
if step7_csv.is_file():
|
||||||
|
return str(step7_csv).replace('\\', '/')
|
||||||
|
|
||||||
|
# 2) 10_WaterIndex_CSV 下任何带 "training" 关键词的 csv(用户在 Step 10 跑过训练集)
|
||||||
|
idx_dir = wd / "10_WaterIndex_CSV"
|
||||||
|
if idx_dir.is_dir():
|
||||||
|
candidates = sorted(
|
||||||
|
idx_dir.glob("*training*.csv"),
|
||||||
|
key=lambda p: p.stat().st_mtime,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
if candidates:
|
||||||
|
return str(candidates[0]).replace('\\', '/')
|
||||||
|
|
||||||
|
# 3) Step 6 原始光谱特征
|
||||||
|
step6_csv = wd / "6_Spectral_Feature_Extraction" / "training_spectra.csv"
|
||||||
|
if step6_csv.is_file():
|
||||||
|
return str(step6_csv).replace('\\', '/')
|
||||||
|
|
||||||
|
# 4) Step 7 目录下任何 training*.csv(兜底)
|
||||||
|
step7_dir = wd / "7_Water_Quality_Indices"
|
||||||
|
if step7_dir.is_dir():
|
||||||
|
candidates = sorted(
|
||||||
|
step7_dir.glob("*training*.csv"),
|
||||||
|
key=lambda p: p.stat().st_mtime,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
if candidates:
|
||||||
|
return str(candidates[0]).replace('\\', '/')
|
||||||
|
|
||||||
|
return ""
|
||||||
|
|
||||||
def browse_output_path(self):
|
def browse_output_path(self):
|
||||||
"""浏览输出模型目录"""
|
"""浏览输出模型目录"""
|
||||||
work_dir = getattr(self, 'work_dir', "")
|
work_dir = getattr(self, 'work_dir', "")
|
||||||
@ -342,7 +477,8 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
]
|
]
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
'feature_start_column': self.feature_start.text(),
|
# QComboBox 适配:currentText() 拿用户选的列名(与原 QLineEdit.text() 语义等价)
|
||||||
|
'feature_start_column': self.feature_start.currentText(),
|
||||||
'preprocessing_methods': preprocessing_methods if preprocessing_methods else ['None'],
|
'preprocessing_methods': preprocessing_methods if preprocessing_methods else ['None'],
|
||||||
'model_names': model_names if model_names else ['SVR'],
|
'model_names': model_names if model_names else ['SVR'],
|
||||||
'split_methods': split_methods if split_methods else ['random'],
|
'split_methods': split_methods if split_methods else ['random'],
|
||||||
@ -359,7 +495,15 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
def set_config(self, config):
|
def set_config(self, config):
|
||||||
"""设置配置"""
|
"""设置配置"""
|
||||||
if 'feature_start_column' in config:
|
if 'feature_start_column' in config:
|
||||||
self.feature_start.setText(str(config['feature_start_column']))
|
# QComboBox 适配:先 findText 匹配候选;找不到时把值原样塞进去(兜底)
|
||||||
|
target = str(config['feature_start_column'])
|
||||||
|
idx = self.feature_start.findText(target)
|
||||||
|
if idx >= 0:
|
||||||
|
self.feature_start.setCurrentIndex(idx)
|
||||||
|
else:
|
||||||
|
# 配置回放时 CSV 可能尚未选定,候选列表为空;先插一项保留语义
|
||||||
|
self.feature_start.addItem(target, target)
|
||||||
|
self.feature_start.setCurrentIndex(self.feature_start.count() - 1)
|
||||||
if 'cv_folds' in config:
|
if 'cv_folds' in config:
|
||||||
self.cv_folds.setValue(config['cv_folds'])
|
self.cv_folds.setValue(config['cv_folds'])
|
||||||
if 'preprocessing_methods' in config:
|
if 'preprocessing_methods' in config:
|
||||||
@ -393,20 +537,15 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
else:
|
else:
|
||||||
self.work_dir = None
|
self.work_dir = None
|
||||||
|
|
||||||
# 1. 强制读 Step 6 的 training_spectra.csv(光谱特征提取结果)
|
# 1. 智能挑选训练 CSV(不再"强制"读 Step 6,而是优先 WQI 增强版)
|
||||||
# 修复张冠李戴:原链路 STEP_DATA_SOURCE['training_spectra_csv'] → step5_clean_panel
|
# 优先级:Step 7 WQI > 10_WaterIndex_CSV/*training* > Step 6 原始光谱 > Step 7 兜底
|
||||||
# 错误地指向了 Step 5 的 processed_data.csv(纯清洗数据,不含光谱特征),
|
# 修复目标:用户跑过 Step 7 后再回到 Step 8,UI 默认应指向带指数的训练集,
|
||||||
# 实际 ML 训练需要的特征数据来自 Step 6 的 6_Spectral_Feature_Extraction/training_spectra.csv
|
# 否则训练好的模型有 95 维(50 波段 + 45 WQI),下次回放变成只 50 维训练,特征维数错位。
|
||||||
main_window = self.window()
|
|
||||||
existing_training_csv = self.training_csv_file.get_path()
|
existing_training_csv = self.training_csv_file.get_path()
|
||||||
if not existing_training_csv or not existing_training_csv.strip():
|
if not existing_training_csv or not existing_training_csv.strip():
|
||||||
if self.work_dir:
|
candidate = self._resolve_training_csv_from_workdir()
|
||||||
step6_dir = resolve_subdir(self.work_dir, 'spectral_feature')
|
if candidate:
|
||||||
step6_training_csv = os.path.join(
|
self.training_csv_file.set_path(candidate)
|
||||||
step6_dir, 'training_spectra.csv'
|
|
||||||
).replace('\\', '/')
|
|
||||||
if step6_training_csv:
|
|
||||||
self.training_csv_file.set_path(step6_training_csv)
|
|
||||||
|
|
||||||
# 2. 自动填充输出目录为 8_Machine_Learning_Models
|
# 2. 自动填充输出目录为 8_Machine_Learning_Models
|
||||||
if self.work_dir:
|
if self.work_dir:
|
||||||
@ -448,7 +587,9 @@ class Step8MlTrainPanel(QWidget):
|
|||||||
"""获取模型训练参数"""
|
"""获取模型训练参数"""
|
||||||
return {
|
return {
|
||||||
'pipeline_type': 'machine_learning',
|
'pipeline_type': 'machine_learning',
|
||||||
'feature_start': float(self.feature_start.text()),
|
# QComboBox 适配:currentText() 取列名;下拉项里就是"374.285004" 等纯数字波段名
|
||||||
|
# (与原 QLineEdit 中 "374.285004" 字符串保持完全一致,后端 float() 解析不变)
|
||||||
|
'feature_start': float(self.feature_start.currentText()),
|
||||||
'cv_folds': self.cv_folds.value(),
|
'cv_folds': self.cv_folds.value(),
|
||||||
'preprocess_methods': [method for method, cb in self.preproc_checkboxes.items() if cb.isChecked()],
|
'preprocess_methods': [method for method, cb in self.preproc_checkboxes.items() if cb.isChecked()],
|
||||||
'model_types': [model for model, cb in self.model_checkboxes.items() if cb.isChecked()],
|
'model_types': [model for model, cb in self.model_checkboxes.items() if cb.isChecked()],
|
||||||
|
|||||||
@ -8,6 +8,8 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
# 路径归一化 helper(与 pipeline.get_step_output_dir 互为表里)
|
# 路径归一化 helper(与 pipeline.get_step_output_dir 互为表里)
|
||||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
if _HERE not in sys.path:
|
if _HERE not in sys.path:
|
||||||
@ -333,24 +335,90 @@ class Step9MlPredictPanel(QWidget):
|
|||||||
result[name] = self.external_models_dict[name]
|
result[name] = self.external_models_dict[name]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def _resolve_latest_wqi_test_csv(self):
|
||||||
|
"""在工作目录中智能挑选"最新生成的、含 WQI 指数的测试集 CSV"。
|
||||||
|
|
||||||
|
返回:找到则返回文件路径字符串;找不到返回 ""。
|
||||||
|
|
||||||
|
搜索策略(按优先级递减,命中即返回):
|
||||||
|
1) 10_WaterIndex_CSV/*.csv — Step 10 输出目录(用户在 Step 10 跑过的产品)
|
||||||
|
2) 7_Water_Quality_Indices/*sampling*.csv / *test*.csv — 用户手动对采样点算过 WQI
|
||||||
|
3) work_dir 下任何 *indices*.csv / *wqi*.csv(不区分大小写)
|
||||||
|
4) work_dir 下任何 > 60 列的 csv(启发式:50 波段 + > 10 WQI 指数列)
|
||||||
|
5) 兜底空串,调用方回退到 Step 4 sampling_spectra.csv
|
||||||
|
|
||||||
|
多个候选时按 mtime 倒序选"最新生成的"。
|
||||||
|
"""
|
||||||
|
work_dir = self._get_default_work_dir()
|
||||||
|
if not work_dir:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
wd = Path(work_dir)
|
||||||
|
found = []
|
||||||
|
|
||||||
|
# 1) 10_WaterIndex_CSV 下所有 csv(Step 10 输出)
|
||||||
|
idx_dir = wd / "10_WaterIndex_CSV"
|
||||||
|
if idx_dir.is_dir():
|
||||||
|
found.extend(idx_dir.glob("*.csv"))
|
||||||
|
|
||||||
|
# 2) 7_Water_Quality_Indices 下与采样/测试相关的 csv
|
||||||
|
qa_dir = wd / "7_Water_Quality_Indices"
|
||||||
|
if qa_dir.is_dir():
|
||||||
|
for pattern in ("*sampling*.csv", "*test*.csv", "*predict*.csv"):
|
||||||
|
found.extend(qa_dir.glob(pattern))
|
||||||
|
|
||||||
|
# 3) work_dir 直接子树下含 indices/wqi 关键词的 csv
|
||||||
|
for keyword in ("*indices*.csv", "*wqi*.csv", "*WQI*.csv"):
|
||||||
|
found.extend(wd.rglob(keyword))
|
||||||
|
|
||||||
|
# 4) 启发式:> 60 列的 csv(50 波段 + 至少 10 个指数)
|
||||||
|
try:
|
||||||
|
for csv in wd.rglob("*.csv"):
|
||||||
|
if csv in found:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
head = pd.read_csv(csv, nrows=0)
|
||||||
|
if head.shape[1] > 60:
|
||||||
|
found.append(csv)
|
||||||
|
except Exception:
|
||||||
|
pass # 读取失败就跳过,不影响其它候选
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not found:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
# 去重 + 按 mtime 倒序排
|
||||||
|
uniq = {p.resolve(): p for p in found}.values()
|
||||||
|
sorted_paths = sorted(uniq, key=lambda p: p.stat().st_mtime, reverse=True)
|
||||||
|
return str(sorted_paths[0]).replace('\\', '/')
|
||||||
|
|
||||||
def update_from_config(self, work_dir=None, pipeline=None):
|
def update_from_config(self, work_dir=None, pipeline=None):
|
||||||
if work_dir: self.work_dir = work_dir
|
if work_dir: self.work_dir = work_dir
|
||||||
|
|
||||||
main_window = self.window()
|
main_window = self.window()
|
||||||
factory = getattr(main_window, '_panel_factory', None) if main_window else None
|
factory = getattr(main_window, '_panel_factory', None) if main_window else None
|
||||||
if not factory: return
|
|
||||||
|
|
||||||
# 1. 拿第 4 步的采样光谱
|
# 1. 智能挑选采样 CSV:优先"含 WQI 指数的测试集"(防止特征维度与训练时不匹配)
|
||||||
step4_panel = factory.get_panel('step4_sampling')
|
# 修复目标:用户在 Step 8 用 95 维 (50+45 WQI) 训练 → Step 9 默认读 50 维 raw sampling
|
||||||
if step4_panel and hasattr(step4_panel, 'output_file'):
|
# 时 inference_batch.preprocess_spectra 会触发"自动特征补全"逻辑;但若用户已经在
|
||||||
path = step4_panel.output_file.get_path()
|
# Step 10/手工把指数算到 CSV 里了,应该直接用那个文件(少走内存补全、避免 band 列顺序漂移)
|
||||||
if path: self.sampling_csv_file.set_path(path)
|
wqi_test_csv = self._resolve_latest_wqi_test_csv()
|
||||||
|
if wqi_test_csv:
|
||||||
|
self.sampling_csv_file.set_path(wqi_test_csv)
|
||||||
|
elif factory:
|
||||||
|
# 兜底:拿第 4 步的纯原始采样光谱(旧行为保留)
|
||||||
|
step4_panel = factory.get_panel('step4_sampling')
|
||||||
|
if step4_panel and hasattr(step4_panel, 'output_file'):
|
||||||
|
path = step4_panel.output_file.get_path()
|
||||||
|
if path: self.sampling_csv_file.set_path(path)
|
||||||
|
|
||||||
# 2. 拿第 8 步的模型目录
|
# 2. 拿第 8 步的模型目录
|
||||||
step8_panel = factory.get_panel('step8_ml_train')
|
if factory:
|
||||||
if step8_panel and hasattr(step8_panel, 'output_path'):
|
step8_panel = factory.get_panel('step8_ml_train')
|
||||||
path = step8_panel.output_path.get_path()
|
if step8_panel and hasattr(step8_panel, 'output_path'):
|
||||||
if path: self.models_dir_file.set_path(path)
|
path = step8_panel.output_path.get_path()
|
||||||
|
if path: self.models_dir_file.set_path(path)
|
||||||
|
|
||||||
# 3. 生成第 9 步的输出目录
|
# 3. 生成第 9 步的输出目录
|
||||||
if hasattr(self, 'work_dir') and self.work_dir:
|
if hasattr(self, 'work_dir') and self.work_dir:
|
||||||
|
|||||||
Reference in New Issue
Block a user