Compare commits

...

5 Commits

14 changed files with 267 additions and 271 deletions

View File

@ -348,7 +348,7 @@ class WaterIndexProcessor:
hdr_path : str, optional
ENVI HDR 文件路径None → 自动构造)
output_dir : str, optional
输出目录None → 与 bsq_path 同目录下的 8_WaterIndex_Images/
输出目录None → 与 bsq_path 同目录下的 10_WaterIndex_Images/
formula_names : list, optional
要处理的公式名列表None → 处理全部)
water_mask : np.ndarray, optional
@ -374,7 +374,7 @@ class WaterIndexProcessor:
# ── 自动构造输出目录 ────────────────────────────────────────────
if output_dir is None:
output_dir = os.path.join(os.path.dirname(bsq_path), '8_WaterIndex_Images')
output_dir = os.path.join(os.path.dirname(bsq_path), '10_WaterIndex_Images')
os.makedirs(output_dir, exist_ok=True)
def progress(msg: str, pct: float):
@ -610,7 +610,7 @@ class WaterIndexProcessor:
if os.path.isfile(hdr_path_alt):
hdr_path = hdr_path_alt
output_dir = os.path.join(work_dir, "8_WaterIndex_Images")
output_dir = os.path.join(work_dir, "10_WaterIndex_Images")
# ── 加载水域掩膜(可选)───────────────────────────────────────
water_mask: Optional[np.ndarray] = None

View File

@ -2,6 +2,7 @@ from osgeo import gdal, osr
import numpy as np
import pandas as pd
import os
import re
import spectral
from math import sin, cos, tan, sqrt, radians
@ -473,9 +474,56 @@ def get_spectral_in_coor(imgpath, coorpath, outpath, radius=0, flare_path=None,
for i in range(min(3, coor_data.shape[0])):
print(f"{i + 1}: {coor_data[i, :min(5, coor_data.shape[1])]}") # 只显示前5列
# 提取原始坐标
lat_array = coor_data[:, 1] # 第2列是纬度跳过测量点ID列
lon_array = coor_data[:, 2] # 第3列是经度
# 提取原始坐标(使用智能坐标列检测)
lon_patterns = [
r'^lon', r'^lng', r'^longitude', r'经度', r'^x$', r'^utm_x$', r'^pixel_x$'
]
lat_patterns = [
r'^lat', r'^latitude', r'纬度', r'^y$', r'^utm_y$', r'^pixel_y$'
]
x_col_name, y_col_name = None, None
if coor_df is not None and hasattr(coor_df, 'columns'):
for col in coor_df.columns:
col_str = str(col).lower().strip()
if x_col_name is None and any(re.search(p, col_str) for p in lon_patterns):
x_col_name = col
if y_col_name is None and any(re.search(p, col_str) for p in lat_patterns):
y_col_name = col
if x_col_name and y_col_name and x_col_name in coor_df.columns and y_col_name in coor_df.columns:
lon_array = coor_df[x_col_name].values
lat_array = coor_df[y_col_name].values
print(f"💡 坐标列名检测: X/经度=[{x_col_name}], Y/纬度=[{y_col_name}]")
else:
numeric_cols = coor_df.select_dtypes(include=[np.number]).columns.tolist() if coor_df is not None else []
if len(numeric_cols) >= 2:
col1, col2 = numeric_cols[0], numeric_cols[1]
mean1 = coor_df[col1].head(10).mean()
mean2 = coor_df[col2].head(10).mean()
if abs(mean1) <= 90 and abs(mean2) > 90:
y_col_name, x_col_name = col1, col2
lon_array = coor_df[x_col_name].values
lat_array = coor_df[y_col_name].values
elif abs(mean2) <= 90 and abs(mean1) > 90:
x_col_name, y_col_name = col1, col2
lon_array = coor_df[x_col_name].values
lat_array = coor_df[y_col_name].values
else:
if mean1 > mean2:
x_col_name, y_col_name = col1, col2
else:
x_col_name, y_col_name = col2, col1
lon_array = coor_df[x_col_name].values
lat_array = coor_df[y_col_name].values
print(f"💡 触发智能数值推断坐标列: X/经度=[{x_col_name}], Y/纬度=[{y_col_name}]")
else:
if coor_data is not None and coor_data.shape[1] >= 3:
lat_array = coor_data[:, 1]
lon_array = coor_data[:, 2]
else:
raise Exception("坐标文件格式错误需要至少2列数据且最好包含坐标列名如lon/lat/经度/纬度)")
print(f"\n=== 原始坐标信息 ===")
print(f"原始坐标范围: 经度 {np.min(lon_array):.6f} ~ {np.max(lon_array):.6f}, 纬度 {np.min(lat_array):.6f} ~ {np.max(lat_array):.6f}")

View File

@ -247,7 +247,7 @@ def non_empirical_retrieval(algorithm, model_info_path, coor_spectral_path, outp
if __name__ == "__main__":
algorithm= "chl_a"
model_info_path= r"E:\code\WQ\pipeline_result\work_dir\5_training_spectra\8_non_empirical_models\SS\SS_chl_a.json"
coor_spectral_path= r"E:\code\WQ\pipeline_result\work_dir\10_sampling\sampling_spectra.csv"
coor_spectral_path= r"E:\code\WQ\pipeline_result\work_dir\4_sampling\sampling_spectra.csv"
output_path= r"E:\code\WQ\pipeline_result\work_dir\11_12_13_predictions\SS_chl_a.csv"
wave_radius=5.0
non_empirical_retrieval(algorithm, model_info_path, coor_spectral_path, output_path, wave_radius)

View File

@ -24,7 +24,7 @@ class PredictionStep:
chunk_size: int = 1000,
water_mask_path: Optional[str] = None,
glint_mask_path: Optional[str] = None,
output_dir: Union[str, Path] = "./10_sampling",
output_dir: Union[str, Path] = "./4_sampling",
callback: Optional[Callable] = None,
use_adaptive_sampling: bool = True,
) -> str:

View File

@ -144,7 +144,7 @@ class WaterQualityInversionPipeline:
self.models_dir = self.work_dir / "7_Supervised_Model_Training"
self.non_empirical_models_dir = self.work_dir / "8_Regression_Modeling"
self.custom_regression_dir = self.work_dir / "9_Custom_Regression_Modeling"
self.sampling_dir = self.work_dir / "10_sampling"
self.sampling_dir = self.work_dir / "4_sampling"
self.prediction_dir = self.work_dir / "11_12_13_predictions"
self.visualization_dir = self.work_dir / "14_visualization"
self.reports_dir = self.work_dir / "reports"
@ -2276,7 +2276,7 @@ def main():
'step6': {
'formula_csv_file': 'path/to/water_quality_formulas.csv', # 公式CSV文件路径
'formula_names': ['Al10SABI', 'TurbBe16RedOverViolet'], # 要计算的公式名称列表
'output_filename': 'water_quality_indices.csv',
'output_filename': 'training_spectra_indices.csv',
'enabled': True # 是否启用水质指数计算
},
'step7': {

View File

@ -204,7 +204,7 @@ class Step10WatercolorPanel(QWidget):
"输出目录:",
"Directories"
)
self.output_dir.line_edit.setPlaceholderText("留空 → 工作目录/8_WaterIndex_Images")
self.output_dir.line_edit.setPlaceholderText("留空 → 工作目录/10_WaterIndex_Images")
self.output_dir.browse_btn.clicked.disconnect()
self.output_dir.browse_btn.clicked.connect(self._browse_output_dir)
output_layout.addRow("输出目录:", self.output_dir)
@ -437,22 +437,42 @@ class Step10WatercolorPanel(QWidget):
self.work_dir = None
main_window = self.window()
deglint_path = None
# 自动填入去耀斑影像
if main_window and hasattr(main_window, 'step3_panel'):
# 1. 优先从 pipeline 的真实输出中获取
if pipeline and hasattr(pipeline, 'step_outputs'):
step3_out = pipeline.step_outputs.get('step3', {})
deglint_path = step3_out.get('deglint_image') or step3_out.get('output_path')
# 2. 回退:从 step3 面板实例获取
if not deglint_path and main_window and hasattr(main_window, 'step3_panel'):
if hasattr(main_window.step3_panel, 'output_file'):
deglint_path = main_window.step3_panel.output_file.get_path()
if deglint_path and not self.bsq_file.get_path():
# 3. 终极回退:智能扫描 3_deglint 目录,取最新的 .bsq 或 .dat 文件
if not deglint_path and self.work_dir:
deglint_dir = os.path.join(self.work_dir, "3_deglint")
if os.path.isdir(deglint_dir):
import glob
candidates = glob.glob(os.path.join(deglint_dir, "*.bsq")) + glob.glob(os.path.join(deglint_dir, "*.dat"))
if candidates:
candidates.sort(key=os.path.getmtime, reverse=True)
deglint_path = candidates[0]
# 填入 UI 并自动寻找对应的 hdr 文件
if deglint_path:
if not os.path.isabs(deglint_path):
deglint_path = os.path.join(self.work_dir or '', deglint_path).replace('\\', '/')
self.bsq_file.set_path(deglint_path)
hdr = Path(deglint_path).with_suffix('.hdr')
if hdr.exists():
self.hdr_file.set_path(str(hdr))
self._load_metadata(deglint_path, str(hdr))
hdr_path = os.path.splitext(deglint_path)[0] + '.hdr'
if os.path.exists(hdr_path):
self.hdr_file.set_path(hdr_path)
self._load_metadata(deglint_path, hdr_path)
# 自动填入输出目录
if self.work_dir:
out_dir = os.path.join(self.work_dir, "8_WaterIndex_Images").replace('\\', '/')
out_dir = os.path.join(self.work_dir, "10_WaterIndex_Images").replace('\\', '/')
os.makedirs(out_dir, exist_ok=True)
if not self.output_dir.get_path():
self.output_dir.set_path(out_dir)
@ -483,7 +503,7 @@ class Step10WatercolorPanel(QWidget):
return
if not output_dir:
work_dir = self._get_default_work_dir()
output_dir = os.path.join(work_dir, "8_WaterIndex_Images").replace('\\', '/')
output_dir = os.path.join(work_dir, "10_WaterIndex_Images").replace('\\', '/')
os.makedirs(output_dir, exist_ok=True)
self.output_dir.set_path(output_dir)

View File

@ -253,7 +253,7 @@ class Step11MapPanel(QWidget):
self.geotiff_dir_label = QLabel("水色指数目录:")
self.geotiff_dir_label.setMinimumWidth(120)
self.geotiff_dir_edit = QLineEdit()
self.geotiff_dir_edit.setPlaceholderText("选择 8_WaterIndex_Images 文件夹(批量渲染)…")
self.geotiff_dir_edit.setPlaceholderText("选择 10_WaterIndex_Images 文件夹(批量渲染)…")
geotiff_dir_btn = QPushButton("浏览…")
geotiff_dir_btn.setMaximumWidth(80)
geotiff_dir_btn.clicked.connect(self.browse_geotiff_dir)
@ -392,7 +392,7 @@ class Step11MapPanel(QWidget):
"""浏览 GeoTIFF 文件夹(批量模式)"""
default = self._get_default_work_dir()
if default:
default = os.path.join(default, "8_WaterIndex_Images")
default = os.path.join(default, "10_WaterIndex_Images")
d = QFileDialog.getExistingDirectory(
self, "选择水色指数 GeoTIFF 文件夹", default
)
@ -514,18 +514,18 @@ class Step11MapPanel(QWidget):
pred_dir = None
if hasattr(main_window, 'step11_prediction_panel'):
step8_widget = getattr(main_window.step11_prediction_panel, 'output_file', None)
step8_output = ""
step10_output = ""
if hasattr(step8_widget, 'get_path'):
step8_output = step8_widget.get_path() or ""
step10_output = step8_widget.get_path() or ""
elif hasattr(step8_widget, 'text'):
step8_output = step8_widget.text() or ""
step10_output = step8_widget.text() or ""
if step8_output:
if step10_output:
# 若为相对路径,使用 work_dir 合成为绝对路径
if not os.path.isabs(step8_output):
step8_output = os.path.join(self.work_dir or '', step8_output).replace('\\', '/')
if not os.path.isabs(step10_output):
step10_output = os.path.join(self.work_dir or '', step10_output).replace('\\', '/')
# 提取父目录后追加 Machine_Learning_Prediction最底层真实子目录
base_pred_dir = str(Path(step8_output).parent)
base_pred_dir = str(Path(step10_output).parent)
ml_pred_dir = Path(base_pred_dir) / "Machine_Learning_Prediction"
pred_dir = str(ml_pred_dir) if ml_pred_dir.exists() else base_pred_dir
@ -594,13 +594,13 @@ class Step11MapPanel(QWidget):
print("⚠️ 提示:专题图生成模块需传入标准矢量边界文件 (.shp),请手动选择。")
# 6. 自动探测 Step 8 输出的水色指数 GeoTIFFGeoTIFF 渲染模式)
step8_out_dir = Path(self.work_dir) / "8_WaterIndex_Images" if self.work_dir else None
if step8_out_dir and step8_out_dir.is_dir():
step10_out_dir = Path(self.work_dir) / "10_WaterIndex_Images" if self.work_dir else None
if step10_out_dir and step10_out_dir.is_dir():
# GeoTIFF 批量模式:填充目录供批量渲染
if not (self.geotiff_dir_edit.text() or "").strip():
self.geotiff_dir_edit.setText(str(step8_out_dir))
self.geotiff_dir_edit.setText(str(step10_out_dir))
# GeoTIFF 单文件模式:默认选中第一个
tif_files = sorted(step8_out_dir.glob("*.tif"))
tif_files = sorted(step10_out_dir.glob("*.tif"))
if tif_files and not (self.geotiff_file.get_path() or "").strip():
self.geotiff_file.set_path(str(tif_files[0]))
except Exception as e:

View File

@ -253,7 +253,7 @@ class Step14Panel(QWidget):
self.geotiff_dir_label = QLabel("水色指数目录:")
self.geotiff_dir_label.setMinimumWidth(120)
self.geotiff_dir_edit = QLineEdit()
self.geotiff_dir_edit.setPlaceholderText("选择 8_WaterIndex_Images 文件夹(批量渲染)…")
self.geotiff_dir_edit.setPlaceholderText("选择 10_WaterIndex_Images 文件夹(批量渲染)…")
geotiff_dir_btn = QPushButton("浏览…")
geotiff_dir_btn.setMaximumWidth(80)
geotiff_dir_btn.clicked.connect(self.browse_geotiff_dir)
@ -392,7 +392,7 @@ class Step14Panel(QWidget):
"""浏览 GeoTIFF 文件夹(批量模式)"""
default = self._get_default_work_dir()
if default:
default = os.path.join(default, "8_WaterIndex_Images")
default = os.path.join(default, "10_WaterIndex_Images")
d = QFileDialog.getExistingDirectory(
self, "选择水色指数 GeoTIFF 文件夹", default
)
@ -514,18 +514,18 @@ class Step14Panel(QWidget):
pred_dir = None
if hasattr(main_window, 'step11_prediction_panel'):
step8_widget = getattr(main_window.step11_prediction_panel, 'output_file', None)
step8_output = ""
step10_output = ""
if hasattr(step8_widget, 'get_path'):
step8_output = step8_widget.get_path() or ""
step10_output = step8_widget.get_path() or ""
elif hasattr(step8_widget, 'text'):
step8_output = step8_widget.text() or ""
step10_output = step8_widget.text() or ""
if step8_output:
if step10_output:
# 若为相对路径,使用 work_dir 合成为绝对路径
if not os.path.isabs(step8_output):
step8_output = os.path.join(self.work_dir or '', step8_output).replace('\\', '/')
if not os.path.isabs(step10_output):
step10_output = os.path.join(self.work_dir or '', step10_output).replace('\\', '/')
# 提取父目录后追加 Machine_Learning_Prediction最底层真实子目录
base_pred_dir = str(Path(step8_output).parent)
base_pred_dir = str(Path(step10_output).parent)
ml_pred_dir = Path(base_pred_dir) / "Machine_Learning_Prediction"
pred_dir = str(ml_pred_dir) if ml_pred_dir.exists() else base_pred_dir
@ -594,13 +594,13 @@ class Step14Panel(QWidget):
print("⚠️ 提示:专题图生成模块需传入标准矢量边界文件 (.shp),请手动选择。")
# 6. 自动探测 Step 8 输出的水色指数 GeoTIFFGeoTIFF 渲染模式)
step8_out_dir = Path(self.work_dir) / "8_WaterIndex_Images" if self.work_dir else None
if step8_out_dir and step8_out_dir.is_dir():
step10_out_dir = Path(self.work_dir) / "10_WaterIndex_Images" if self.work_dir else None
if step10_out_dir and step10_out_dir.is_dir():
# GeoTIFF 批量模式:填充目录供批量渲染
if not (self.geotiff_dir_edit.text() or "").strip():
self.geotiff_dir_edit.setText(str(step8_out_dir))
self.geotiff_dir_edit.setText(str(step10_out_dir))
# GeoTIFF 单文件模式:默认选中第一个
tif_files = sorted(step8_out_dir.glob("*.tif"))
tif_files = sorted(step10_out_dir.glob("*.tif"))
if tif_files and not (self.geotiff_file.get_path() or "").strip():
self.geotiff_file.set_path(str(tif_files[0]))
except Exception as e:

View File

@ -6,6 +6,7 @@ Step4 面板 - 采样点布设
import os
from PyQt5.QtCore import QTimer
from PyQt5.QtWidgets import (
QWidget, QVBoxLayout, QGroupBox, QFormLayout,
QPushButton, QCheckBox, QSpinBox, QMessageBox,
@ -94,6 +95,11 @@ class Step4SamplingPanel(QWidget):
layout.addStretch()
self.setLayout(layout)
# 添加心跳定时器每2秒自动检查一次输出文件状态刷新预览按钮
self._status_timer = QTimer(self)
self._status_timer.timeout.connect(self._check_csv_exists)
self._status_timer.start(2000)
# 监听输出路径变化,实时更新预览按钮状态
self.output_file.line_edit.textChanged.connect(self._on_output_changed)

View File

@ -7,9 +7,8 @@ Step7 面板 - 水质指数计算
import os
import sys
import pandas as pd
import numpy as np
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from typing import Dict, List, Optional
from PyQt5.QtWidgets import (
QWidget, QVBoxLayout, QGroupBox, QGridLayout,
@ -127,25 +126,10 @@ class Step7IndexPanel(QWidget):
self.formula_group.setLayout(formula_outer_layout)
main_layout.addWidget(self.formula_group)
# 4. 输出选项
output_group = QGroupBox("输出模式")
# 4. 执行设置
output_group = QGroupBox("执行设置")
output_layout = QVBoxLayout()
mode_layout = QHBoxLayout()
self.mode_group = QButtonGroup()
self.radio_both = QRadioButton("两者皆出")
self.radio_wide = QRadioButton("仅宽表")
self.radio_single = QRadioButton("仅单文件")
self.mode_group.addButton(self.radio_both, 0)
self.mode_group.addButton(self.radio_wide, 1)
self.mode_group.addButton(self.radio_single, 2)
self.radio_both.setChecked(True)
mode_layout.addWidget(self.radio_both)
mode_layout.addWidget(self.radio_wide)
mode_layout.addWidget(self.radio_single)
mode_layout.addStretch()
output_layout.addLayout(mode_layout)
self.enable_checkbox = QCheckBox("启用计算流程")
self.enable_checkbox.setChecked(True)
output_layout.addWidget(self.enable_checkbox)
@ -261,25 +245,28 @@ class Step7IndexPanel(QWidget):
for item in self.index_checkboxes.values():
item.setCheckState(Qt.Unchecked)
def get_config(self) -> Dict:
def get_config(self) -> dict:
"""获取配置"""
selected = [
name for name, item in self.index_checkboxes.items()
if item.checkState() == Qt.Checked
]
# Build coefficient dict for selected formulas
formula_coefficients = {
name: self._formula_coef_map.get(name, [])
for name in selected
}
return {
config = {
'training_csv_path': self.training_data_widget.get_path(),
'formula_csv_file': self.builtin_formula_path,
'formula_names': selected,
'formula_coefficients': formula_coefficients,
'enabled': self.enable_checkbox.isChecked(),
'output_mode': self.mode_group.checkedId(),
'output_mode': 0,
}
work_dir = self._get_work_dir()
if work_dir:
track_a_dir = os.path.join(work_dir, "6_water_quality_indices")
os.makedirs(track_a_dir, exist_ok=True)
config['output_file'] = os.path.join(track_a_dir, "training_spectra_indices.csv").replace('\\', '/')
return config
def set_config(self, config: Dict):
if 'training_csv_path' in config:
self.training_data_widget.set_path(config['training_csv_path'])
@ -288,10 +275,6 @@ class Step7IndexPanel(QWidget):
for name, item in self.index_checkboxes.items():
item.setCheckState(Qt.Checked if name in sel else Qt.Unchecked)
self.enable_checkbox.setChecked(config.get('enabled', True))
if 'output_mode' in config:
btn = self.mode_group.button(config['output_mode'])
if btn:
btn.setChecked(True)
def update_from_config(self, work_dir=None, pipeline=None):
if work_dir:
@ -313,117 +296,24 @@ class Step7IndexPanel(QWidget):
return main.work_dir
return None
def _get_coord_cols(self, df: pd.DataFrame) -> Tuple[str, str]:
coord_candidates = ['lon', 'lng', 'longitude', '经度', 'x', 'lon_utm', 'utm_x', 'pixel_x']
lat_candidates = ['lat', 'latitude', '纬度', 'y', 'lat_utm', 'utm_y', 'pixel_y']
x_col, y_col = None, None
for col in df.columns:
cl = col.lower()
if x_col is None and any(c in cl for c in coord_candidates):
x_col = col
if y_col is None and any(c in cl for c in lat_candidates):
y_col = col
if x_col is None and len(df.columns) >= 2:
x_col = df.columns[0]
if y_col is None and len(df.columns) >= 2:
y_col = df.columns[1]
return x_col or 'x_coord', y_col or 'y_coord'
def run_step(self):
"""独立运行步骤7 (通过标准的 Worker 路由下发)"""
config = self.get_config()
if not config['enabled']:
QMessageBox.information(self, "提示", "已禁用计算流程(启用计算流程未勾选)")
return
training_path = config['training_csv_path']
training_path = config.get('training_csv_path')
if not training_path or not os.path.exists(training_path):
QMessageBox.warning(self, "提示", "请先选择输入特征提取CSV文件")
return
formula_names = config['formula_names']
if not formula_names:
if not config.get('formula_names'):
QMessageBox.warning(self, "提示", "请至少勾选一个公式")
return
output_mode = config['output_mode']
try:
from src.core.steps.data_preparation_step import DataPreparationStep
spec_df = pd.read_csv(training_path)
x_col, y_col = self._get_coord_cols(spec_df)
# 构建 formula_csv_path使用内置 waterindex.csv
formula_csv_path = self.builtin_formula_path
if not formula_csv_path or not os.path.exists(formula_csv_path):
# 尝试从 src/gui/model/ 目录找
possible_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'gui', 'model', 'waterindex.csv')
if os.path.exists(possible_path):
formula_csv_path = possible_path
work_dir = self._get_work_dir()
# 调用 DataPreparationStep 的静态方法计算水质指数(宽表输出)
indices_csv_path = DataPreparationStep.calculate_water_quality_indices(
training_csv_path=training_path,
formula_csv_file=formula_csv_path,
formula_names=formula_names,
output_file=None, # 不在此处指定输出,由下面的双轨输出逻辑接管
enabled=True,
output_dir=work_dir if work_dir else os.getcwd(),
)
# 读取计算结果(宽表)
if indices_csv_path and os.path.exists(indices_csv_path):
output_df = pd.read_csv(indices_csv_path)
else:
output_df = spec_df # fallback
track_a_path = None
track_b_dir = None
if output_mode in (0, 1):
track_a_dir = os.path.join(work_dir, "6_water_quality_indices") if work_dir else "6_water_quality_indices"
os.makedirs(track_a_dir, exist_ok=True)
track_a_path = os.path.join(track_a_dir, "training_spectra_indices.csv")
if output_mode in (0, 2):
track_b_dir = os.path.join(work_dir, "11_12_13_predictions", "Traditional_Indices") if work_dir else "11_12_13_predictions/Traditional_Indices"
os.makedirs(track_b_dir, exist_ok=True)
saved = []
if output_mode in (0, 1):
output_df.to_csv(track_a_path, index=False, float_format='%.6f')
saved.append(f"宽表: {track_a_path}")
if output_mode in (0, 2):
coord_x = spec_df[x_col].values if x_col in spec_df.columns else np.arange(len(spec_df))
coord_y = spec_df[y_col].values if y_col in spec_df.columns else np.zeros(len(spec_df))
for formula_name in formula_names:
if formula_name not in output_df.columns:
continue
single_df = pd.DataFrame({
'x_coord': coord_x,
'y_coord': coord_y,
'value': output_df[formula_name].values,
})
safe_name = formula_name.replace('/', '_').replace(' ', '_')
out_path = os.path.join(track_b_dir, f"{safe_name}_prediction.csv")
single_df.to_csv(out_path, index=False, float_format='%.6f')
saved.append(f"单文件目录: {track_b_dir}")
QMessageBox.information(
self, "计算完成",
f"已保存 {len(saved)} 个输出目标:\n" + "\n".join(saved)
)
except ImportError as e:
QMessageBox.critical(self, "依赖错误", f"无法导入模块:\n{e}")
except Exception as e:
import traceback
QMessageBox.critical(self, "计算失败", f"原因: {str(e)}\n{traceback.format_exc()}")
main_window = self.window()
if hasattr(main_window, 'run_single_step'):
pipeline_config = {'step7_index': config}
main_window.run_single_step('step7_index', pipeline_config)

View File

@ -1365,46 +1365,21 @@ class WaterQualityGUI(QMainWindow):
"""初始化步骤依赖关系和标准输出路径"""
# 定义每个步骤的标准输出路径模式(相对于工作目录)
self.step_default_outputs = {
'step1': {
'water_mask_ndwi': '1_water_mask/water_mask_from_ndwi.dat',
'water_mask_shp': '1_water_mask/water_mask_from_shp.dat',
'hsi_preview': '1_water_mask/hsi_preview.png',
'water_mask_overlay': '1_water_mask/water_mask_overlay.png'
},
'step2': {
'glint_mask': '2_glint/severe_glint_area.dat'
},
'step3': {
'deglint_kutser': '3_deglint/deglint_kutser.bsq',
'deglint_goodman': '3_deglint/deglint_goodman.bsq',
'deglint_hedley': '3_deglint/deglint_hedley.bsq',
'deglint_sugar': '3_deglint/deglint_sugar.bsq',
'deglint_interpolated': '3_deglint/interpolated_*.bsq'
},
'step5_clean': {
'processed_data': '4_processed_data/processed_data.csv'
},
'step6_feature': {
'training_spectra': '5_training_spectra/training_spectra.csv'
},
'step7_index': {
'water_indices': '6_water_quality_indices/water_quality_indices.csv'
},
'step8_ml_train': {
'models': '7_Supervised_Model_Training/'
},
'step4_sampling': {
'sampling_points': '10_sampling/sampling_spectra.csv'
},
'step9_ml_predict': {
'predictions': '11_12_13_predictions/Machine_Learning_Prediction/'
},
'step11_map': {
'distribution_maps': '14_visualization/'
}
'step1': "1_water_mask/water_mask_from_ndwi.dat",
'step2': "2_glint/severe_glint_area.dat",
'step3': "3_deglint/deglint_kutser.bsq",
'step4_sampling': "4_sampling/sampling_spectra.csv",
'step5_clean': "4_processed_data/processed_data.csv",
'step6_feature': "5_training_spectra/training_spectra.csv",
'step7_index': "6_water_quality_indices/training_spectra_indices.csv",
'step8_ml_train': "7_Supervised_Model_Training/",
'step9_ml_predict': "11_12_13_predictions/Machine_Learning_Prediction/",
'step10_watercolor': "10_WaterIndex_Images/",
'step11_map': "14_visualization/"
}
# 定义步骤间的依赖关系:{当前步骤: {输入字段: (依赖步骤, 输出类型, 面板属性名)}}
# 依赖关系字典结构:
# '当前步骤ID': { '依赖参数名': ('上游步骤ID', '上游输出类型/Key', '当前步骤接收该路径的组件属性名') }
self.step_dependencies = {
'step2': {
'img_path': ('step1', 'reference_img', 'img_file'),
@ -1412,31 +1387,36 @@ class WaterQualityGUI(QMainWindow):
},
'step3': {
'img_path': ('step1', 'reference_img', 'img_file'),
'water_mask': ('step1', 'water_mask', 'water_mask_file'),
'water_mask': ('step1', 'water_mask', 'water_mask_file')
},
'step4_sampling': {
'deglint_img_path': ('step3', 'deglint_image', 'deglint_img_file'),
'water_mask_path': ('step1', 'water_mask', 'water_mask_file')
},
'step5_clean': {
'csv_path': ('step4_sampling', 'sampling_spectra', 'csv_file') # step5 寻找 step4 的采样点
},
'step6_feature': {
'deglint_img_path': ('step3', 'deglint_image', 'deglint_img_file'),
'csv_path': ('step5_clean', 'processed_data', 'csv_file'),
'boundary_mask_path': ('step1', 'water_mask', 'boundary_mask_file'),
'boundary_mask_path': ('step1', 'water_mask', 'water_mask_file'), # step6_panel里叫water_mask_file
'glint_mask_path': ('step2', 'glint_mask', 'glint_mask_file')
},
'step7_index': {
'training_csv_path': ('step6_feature', 'training_spectra', 'output_file')
'training_csv_path': ('step6_feature', 'training_spectra', 'training_data_widget') # step7 找 step6 的光谱提取
},
'step8_ml_train': {
'training_csv_path': ('step7_index', 'water_indices', 'csv_file')
},
'step4_sampling': {
'deglint_img_path': ('step3', 'deglint_image', 'deglint_img_file'),
'water_mask_path': ('step1', 'water_mask', 'water_mask_file'),
'glint_mask_path': ('step2', 'glint_mask', 'glint_mask_file')
'training_csv_file': ('step7_index', 'training_spectra_indices', 'training_csv_file') # step8 找 step7 的指数宽表
},
'step9_ml_predict': {
'sampling_csv_path': ('step4_sampling', 'sampling_points', 'sampling_csv_file'),
'models_dir': ('step8_ml_train', 'models', 'models_dir_file')
'models_dir': ('step8_ml_train', 'Supervised_Model_Training', 'models_dir_widget')
},
'step10_watercolor': {
'bsq_file': ('step3', 'deglint_image', 'bsq_file') # 水色反演需要去耀斑BSQ影像
},
'step11_map': {
'prediction_csv_path': ('step9_ml_predict', 'predictions', 'prediction_csv_file')
'prediction_csv_dir_edit': ('step9_ml_predict', 'Machine_Learning_Prediction', 'prediction_csv_dir_edit'),
'geotiff_dir_edit': ('step10_watercolor', 'WaterIndex_Images', 'geotiff_dir_edit')
}
}
@ -2312,7 +2292,17 @@ class WaterQualityGUI(QMainWindow):
if step_id not in self.step_default_outputs:
return None
step_outputs = self.step_default_outputs[step_id]
raw = self.step_default_outputs[step_id]
# ★ 兼容扁平化后的纯字符串路径格式
rel_path = None
if isinstance(raw, str):
rel_path = raw
elif isinstance(raw, dict):
rel_path = raw.get(output_type) or list(raw.values())[0]
if not rel_path:
return None
# ★ 掩膜类型列表:这些类型只接受科学数据格式
mask_types = {'water_mask', 'glint_mask', 'boundary_mask'}
@ -2345,10 +2335,9 @@ class WaterQualityGUI(QMainWindow):
# 根据输出类型查找对应的文件
if output_type == 'water_mask':
# 水域掩膜:优先查找NDWI生成的其次是shp生成的
for mask_type in ['water_mask_ndwi', 'water_mask_shp']:
if mask_type in step_outputs:
mask_path = work_path / step_outputs[mask_type]
# 水域掩膜:直接用统一路径
if rel_path:
mask_path = work_path / rel_path
if mask_path.exists():
return str(mask_path)
elif output_type == 'reference_img':
@ -2358,11 +2347,9 @@ class WaterQualityGUI(QMainWindow):
if img_path and Path(img_path).exists():
return img_path
elif output_type == 'deglint_image':
# 去耀斑影像:查找step3的各种去耀斑方法输出
deglint_types = ['deglint_kutser', 'deglint_goodman', 'deglint_hedley', 'deglint_sugar']
for deglint_type in deglint_types:
if deglint_type in step_outputs:
deglint_path = work_path / step_outputs[deglint_type]
# 去耀斑影像:直接用统一路径
if rel_path:
deglint_path = work_path / rel_path
if deglint_path.exists():
return str(deglint_path)
# 还要检查插值方法生成的文件
@ -2370,17 +2357,16 @@ class WaterQualityGUI(QMainWindow):
if deglint_dir.exists():
for file_path in deglint_dir.glob("interpolated_*.bsq"):
return str(file_path)
elif output_type in step_outputs:
# 直接匹配的输出类型
relative_path = step_outputs[output_type]
if relative_path.endswith('/'):
elif rel_path:
# 直接匹配的输出类型(统一使用 rel_path
if rel_path.endswith('/'):
# 是目录
output_path = work_path / relative_path.rstrip('/')
output_path = work_path / rel_path.rstrip('/')
if output_path.exists() and output_path.is_dir():
return str(output_path)
else:
# 是文件
output_path = work_path / relative_path
output_path = work_path / rel_path
if output_path.exists():
return str(output_path)

View File

@ -4,6 +4,7 @@ if not hasattr(threading.Thread, "isAlive"):
import warnings
import os
import re
import numpy as np
import pandas as pd
from scipy import stats
@ -11,6 +12,54 @@ from scipy import stats
warnings.filterwarnings("ignore")
def auto_detect_coord_columns(df: pd.DataFrame):
"""
双重验证智能识别坐标列:
1. 严格正则匹配列名
2. 基于数值范围的地理学推断
"""
lon_patterns = [
r'^lon', r'^lng', r'^longitude', r'经度', r'^x$', r'^utm_x$', r'^pixel_x$'
]
lat_patterns = [
r'^lat', r'^latitude', r'纬度', r'^y$', r'^utm_y$', r'^pixel_y$'
]
x_col, y_col = None, None
for col in df.columns:
col_str = str(col).lower().strip()
if x_col is None and any(re.search(p, col_str) for p in lon_patterns):
x_col = col
if y_col is None and any(re.search(p, col_str) for p in lat_patterns):
y_col = col
if x_col and y_col:
return x_col, y_col
numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
if len(numeric_cols) >= 2:
col1, col2 = numeric_cols[0], numeric_cols[1]
mean1 = df[col1].head(10).mean()
mean2 = df[col2].head(10).mean()
if abs(mean1) <= 90 and abs(mean2) > 90:
y_col, x_col = col1, col2
elif abs(mean2) <= 90 and abs(mean1) > 90:
x_col, y_col = col1, col2
else:
if mean1 > mean2:
x_col, y_col = col1, col2
else:
x_col, y_col = col2, col1
print(f"💡 触发智能数值推断坐标列: X/经度->[{x_col}], Y/纬度->[{y_col}]")
return x_col, y_col
return df.columns[0], df.columns[1]
def detect_outliers_iqr(data: pd.DataFrame, column: str) -> pd.Series:
"""使用 IQR 方法检测异常值,返回与 data 同索引的布尔序列"""
s = pd.to_numeric(data[column], errors="coerce")
@ -92,11 +141,15 @@ def process_water_quality_data(input_file: str, output_file: str):
print(f"原始数据形状: {df.shape}")
print(f"列名: {list(df.columns)}")
# 0.5) 智能检测坐标列
x_col, y_col = auto_detect_coord_columns(df)
print(f"坐标列检测结果: X/经度=[{x_col}], Y/纬度=[{y_col}]")
# 1) 经纬度精度筛选(小数位 >= 7
print("\n正在筛选经纬度精度(小数位>=7)...")
initial_count = len(df)
for col in ["经度", "纬度"]:
for col in [y_col, x_col]:
if col in df.columns:
dec_len = df[col].apply(_decimal_len)
keep_mask = dec_len >= 7
@ -109,13 +162,11 @@ def process_water_quality_data(input_file: str, output_file: str):
# 2) 异常值检测IQR- 只删除异常值,不删除整行
print("\n正在检测异常值(IQR)...")
# 数值列
numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
# 排除不检测的列
exclude_columns = ["时间", "测量点", "纬度", "经度"]
exclude_columns = {"时间", "测量点", y_col, x_col}
if "原始" in df.columns:
exclude_columns.append("原始")
exclude_columns.add("原始")
columns_to_check = [c for c in numeric_columns if c not in exclude_columns]
print(f"将检测以下列的异常值: {columns_to_check}")
@ -128,7 +179,6 @@ def process_water_quality_data(input_file: str, output_file: str):
col_mask = detect_outliers_iqr(df_clean, column)
outlier_count = int(col_mask.sum())
print(f'"{column}" 检测到 {outlier_count} 个异常值,将其设为 NaN')
# 只将异常值设为 NaN不删除整行
df_clean.loc[col_mask, column] = np.nan
total_outliers_removed += outlier_count
@ -140,7 +190,7 @@ def process_water_quality_data(input_file: str, output_file: str):
df_clean = df_clean.drop(columns=["原始"])
print('已去除 "原始"')
# 4) 字段类型处理:尽量把时间转为 datetime
# 4) 字段类型处理:尽量把"时间"转为 datetime
if "时间" in df_clean.columns:
try:
df_clean["时间"] = pd.to_datetime(df_clean["时间"], errors="coerce")
@ -153,22 +203,18 @@ def process_water_quality_data(input_file: str, output_file: str):
print('错误:未找到 "测量点"')
return
# 构建聚合字典
agg_dict = {}
if "时间" in df_clean.columns and np.issubdtype(df_clean["时间"].dtype, np.datetime64):
# 时间取平均(等价于时间戳平均)
agg_dict["时间"] = "mean"
elif "时间" in df_clean.columns:
# 如果不是时间类型,保留最常见值以避免无意义的字符串平均
agg_dict["时间"] = lambda s: s.mode().iloc[0] if not s.mode().empty else s.dropna().iloc[0] if s.dropna().size else np.nan
for col in ["纬度", "经度"]:
for col in [y_col, x_col]:
if col in df_clean.columns:
agg_dict[col] = "mean"
# 其余数值列取均值
for col in df_clean.select_dtypes(include=[np.number]).columns:
if col not in ["纬度", "经度"]:
if col not in {y_col, x_col}:
agg_dict[col] = "mean"
grouped = df_clean.groupby("测量点", as_index=False).agg(agg_dict)

View File

@ -1049,7 +1049,7 @@ if __name__ == "__main__":
bil_file = r"E:\wq_gui_test\3_deglint\deglint_goodman.bsq"
water_mask_shp = r"E:\wq_gui_test\1_water_mask\water_mask_from_shp.dat"
severe_glint = r"E:\wq_gui_test\2_glint\severe_glint_area.dat"
output_csvpath = r"E:\wq_gui_test\10_sampling\sampling_spectra.csv"
output_csvpath = r"E:\wq_gui_test\4_sampling\sampling_spectra.csv"
# 设置参数
interval = 50 # 基础采样点间隔像元数当use_adaptive_sampling=False时使用

View File

@ -204,7 +204,7 @@ class Step11WaterColorPanel(QWidget):
"输出目录:",
"Directories"
)
self.output_dir.line_edit.setPlaceholderText("留空 → 工作目录/8_WaterIndex_Images")
self.output_dir.line_edit.setPlaceholderText("留空 → 工作目录/10_WaterIndex_Images")
self.output_dir.browse_btn.clicked.disconnect()
self.output_dir.browse_btn.clicked.connect(self._browse_output_dir)
output_layout.addRow("输出目录:", self.output_dir)
@ -452,7 +452,7 @@ class Step11WaterColorPanel(QWidget):
# 自动填入输出目录
if self.work_dir:
out_dir = os.path.join(self.work_dir, "8_WaterIndex_Images").replace('\\', '/')
out_dir = os.path.join(self.work_dir, "10_WaterIndex_Images").replace('\\', '/')
os.makedirs(out_dir, exist_ok=True)
if not self.output_dir.get_path():
self.output_dir.set_path(out_dir)
@ -483,7 +483,7 @@ class Step11WaterColorPanel(QWidget):
return
if not output_dir:
work_dir = self._get_default_work_dir()
output_dir = os.path.join(work_dir, "8_WaterIndex_Images").replace('\\', '/')
output_dir = os.path.join(work_dir, "10_WaterIndex_Images").replace('\\', '/')
os.makedirs(output_dir, exist_ok=True)
self.output_dir.set_path(output_dir)