duxingchen/WQ_GUI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

格式统一

Browse Source
This commit is contained in:
duxin
2026-07-01 09:57:27 +08:00
parent c793ea2204
commit a3c20d3e49
37 changed files with 2286 additions and 1978 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/core/algorithms/waterindex_inversion/csv_processor.py
View File

@ -185,8 +185,10 @@ class WaterIndexCsvProcessor:
missing = [n for n in selected_formulas if n not in all_formula_names]
if missing:
print(f"[WaterIndexCsvProcessor] 警告: 以下公式未在 waterindex.csv 中找到,已跳过: {missing}")
notify(f"用户选择了 {len(selected_formulas)} 个公式 → 匹配到 {len(targets)} 个可计算 ({len(all_formula_names)} 个可用)", 17)
else:
targets = all_formula_names
notify(f"未指定公式 → 计算全部 {len(targets)} 个公式", 17)
if not targets:
raise ValueError("没有可计算的公式（selected_formulas 为空且 waterindex.csv 中无公式）")
@ -202,7 +204,12 @@ class WaterIndexCsvProcessor:
# 每个公式一个 CSV：longitude, latitude, <formula_name>
out_files: Dict[str, str] = {}
n_total = len(targets)
skipped_count = 0
for i, name in enumerate(targets):
# ── 硬守卫：用户显式选择了公式时，绝不输出不在勾选列表中的公式 ──
if selected_formulas and name not in selected_formulas:
skipped_count += 1
continue
try:
per_idx = results_df[name]
# ===== P0 防御: 写盘前清洗（防 Step 11 Kriging / TIN 碎玻璃）=====
@ -239,5 +246,8 @@ class WaterIndexCsvProcessor:
print(f"[WaterIndexCsvProcessor] 公式 '{name}' 失败: {e}")
continue
notify(f"完成！共输出 {len(out_files)} / {n_total} 个指数 CSV", 100)
summary = f"完成！共输出 {len(out_files)} / {n_total} 个指数 CSV"
if skipped_count > 0:
summary += f"，已硬跳过 {skipped_count} 个未勾选公式"
notify(summary, 100)
return out_files

4
src/core/handlers/__init__.py
View File

@ -21,7 +21,9 @@ from src.core.handlers.step7_calc_indices import Step7CalcIndicesHandler
from src.core.handlers.step8_ml_train import Step8MlTrainHandler
from src.core.handlers.step9_ml_predict import Step9MlPredictHandler
from src.core.handlers.step10_qaa_inversion import Step10QaaInversionHandler
from src.core.handlers.step10_watercolor_handler import Step10WatercolorHandler
from src.core.handlers.step11_concentration import Step11ConcentrationHandler
from src.core.handlers.step11_map_handler import Step11MapHandler
from src.core.handlers.step12_kriging import Step12KrigingHandler
from src.core.handlers.step13_visualization import Step13VisualizationHandler
from src.core.handlers.step14_report import Step14ReportHandler
@ -39,7 +41,9 @@ __all__ = [
'Step8MlTrainHandler',
'Step9MlPredictHandler',
'Step10QaaInversionHandler',
'Step10WatercolorHandler',
'Step11ConcentrationHandler',
'Step11MapHandler',
'Step12KrigingHandler',
'Step13VisualizationHandler',
'Step14ReportHandler',

75
src/core/handlers/base.py
View File

@ -43,25 +43,26 @@ class PipelineContext:
self.work_dir = Path(work_dir)
self.work_dir.mkdir(parents=True, exist_ok=True)
# ── 子目录 ──
# ── 子目录（对齐 12 步现代 pipeline）──
self.water_mask_dir = self.work_dir / "1_water_mask"
self.glint_dir = self.work_dir / "2_Glint_Detection"
self.deglint_dir = self.work_dir / "3_deglint"
self.sampling_dir = self.work_dir / "4_sampling"
self.processed_data_dir = self.work_dir / "5_Data_Cleaning"
self.training_spectra_dir = self.work_dir / "6_Spectral_Feature_Extraction"
self.indices_dir = self.work_dir / "7_Water_Quality_Indices"
self.models_dir = self.work_dir / "8_Supervised_Model_Training"
self.non_empirical_models_dir = self.work_dir / "8_Non_Empirical_Regression"
self.custom_regression_dir = self.work_dir / "13_Custom_Regression"
self.sampling_dir = self.work_dir / "4_sampling"
self.prediction_dir = self.work_dir / "11_12_13_predictions"
self.visualization_dir = self.work_dir / "14_visualization"
self.ml_prediction_dir = self.work_dir / "9_ML_Prediction"
self.watercolor_dir = self.work_dir / "10_WaterIndex_CSV"
self.thematic_map_dir = self.work_dir / "11_Thematic_Map"
self.visualization_dir = self.work_dir / "12_visualization"
self.reports_dir = self.work_dir / "reports"
for d in [self.water_mask_dir, self.glint_dir, self.deglint_dir,
self.processed_data_dir, self.training_spectra_dir,
self.indices_dir, self.models_dir, self.non_empirical_models_dir,
self.custom_regression_dir, self.sampling_dir, self.prediction_dir,
self.sampling_dir, self.processed_data_dir,
self.training_spectra_dir, self.indices_dir,
self.models_dir, self.ml_prediction_dir,
self.watercolor_dir, self.thematic_map_dir,
self.visualization_dir, self.reports_dir]:
d.mkdir(parents=True, exist_ok=True)
@ -73,7 +74,6 @@ class PipelineContext:
self.processed_csv_path: Optional[str] = None
self.training_csv_path: Optional[str] = None
self.indices_path: Optional[str] = None
self.custom_regression_path: Optional[str] = None
self.sampling_csv_path: Optional[str] = None
self.prediction_files: Dict[str, str] = {}
self.distribution_map_path: Optional[str] = None
@ -88,6 +88,9 @@ class PipelineContext:
# ── 回调 ──
self._callback: Optional[Callable] = None
# ★ 取消标志（2026-06-30）：支持 Pipeline 优雅中断
self._cancelled: bool = False
# ── 可视化组件（延迟导入避免循环依赖）──
self._visualizer = None
self._report_generator = None
@ -98,6 +101,19 @@ class PipelineContext:
'DejaVu Sans', 'Arial Unicode MS']
plt.rcParams['axes.unicode_minus'] = False
# ═══════════════════════════════════════════════════════════
# 取消机制
# ═══════════════════════════════════════════════════════════
def cancel(self):
"""请求取消当前 Pipeline 执行。"""
self._cancelled = True
self.notify('pipeline', 'info', '收到取消请求，正在终止...')
def is_cancelled(self) -> bool:
"""检查是否已被取消。Handler 可在长时间运算中轮询此标志。"""
return self._cancelled
# ═══════════════════════════════════════════════════════════
# 回调
# ═══════════════════════════════════════════════════════════
@ -193,25 +209,28 @@ class PipelineContext:
'step6_feature': wp / '6_Spectral_Feature_Extraction',
'step7_index': wp / '7_Water_Quality_Indices',
'step8_ml_train': wp / '8_Supervised_Model_Training',
'step9_ml_predict': wp / '8_Non_Empirical_Regression',
'step10_watercolor': wp / '10_WaterIndex_Images',
'step11_map': wp / '14_visualization',
'step12_viz': wp / '14_visualization',
'step13_report': wp / '14_visualization',
'step11_predictions': wp / '11_12_13_predictions',
'step12_predictions': wp / '11_12_13_predictions',
'step13_predictions': wp / '11_12_13_predictions',
'custom_regression': wp / '13_Custom_Regression',
'prediction_dir': wp / '11_12_13_predictions',
'visualization': wp / '14_visualization',
'reports': wp / 'reports',
'step9_ml_predict': wp / '9_ML_Prediction',
'step10_watercolor': wp / '10_WaterIndex_CSV',
'step11_map': wp / '11_Thematic_Map',
'step12_viz': wp / '12_visualization',
'step13_report': wp / '12_visualization',
# 短别名（兼容旧代码按数字索引查找）
'step1': wp / '1_water_mask',
'step2': wp / '2_Glint_Detection',
'step3': wp / '3_deglint',
'step4': wp / '4_sampling',
'step5': wp / '5_Data_Cleaning',
'step6': wp / '6_Spectral_Feature_Extraction',
'step7': wp / '7_Water_Quality_Indices',
'step8': wp / '8_Supervised_Model_Training',
'step9': wp / '8_Non_Empirical_Regression',
'step10': wp / '10_WaterIndex_Images',
'step11': wp / '11_12_13_predictions',
'step12': wp / '13_Custom_Regression',
'step13': wp / 'reports',
'step14': wp / '14_visualization',
'step9': wp / '9_ML_Prediction',
'step10': wp / '10_WaterIndex_CSV',
'step11': wp / '11_Thematic_Map',
'step12': wp / '12_visualization',
'step13': wp / '12_visualization',
# 语义别名
'visualization': wp / '12_visualization',
'reports': wp / 'reports',
}
PipelineContext._STEP_OUTPUT_DIR_MAP = m
return m

18
src/core/handlers/pipeline_scheduler.py
View File

@ -66,6 +66,14 @@ class PipelineScheduler:
for h in handlers:
self.register_handler(h)
# ═══════════════════════════════════════════════════════════
# 取消机制
# ═══════════════════════════════════════════════════════════
def cancel(self):
"""请求取消当前 Pipeline 执行（代理到 PipelineContext）。"""
self.ctx.cancel()
# ═══════════════════════════════════════════════════════════
# 回调
# ═══════════════════════════════════════════════════════════
@ -135,6 +143,11 @@ class PipelineScheduler:
# 按 config 中的顺序遍历（Python 3.7+ dict 保序）
for step_key, step_config in config.items():
# ★ 每步执行前检查取消标志
if self.ctx.is_cancelled():
self.ctx.notify(step_key, 'skipped', '流程已被用户取消')
break
handler = self._handlers.get(step_key)
if handler is None:
@ -152,6 +165,11 @@ class PipelineScheduler:
self.ctx.notify(step_key, 'error', error_msg)
# 不中止，继续执行后续步骤
# ★ 每步执行后也检查取消标志
if self.ctx.is_cancelled():
self.ctx.notify('pipeline', 'info', '流程已被用户取消')
break
self.ctx.pipeline_end_time = time.time()
total_elapsed = self.ctx.pipeline_end_time - self.ctx.pipeline_start_time

4
src/core/handlers/register_handlers.py
View File

@ -21,7 +21,9 @@ from src.core.handlers.step7_calc_indices import Step7CalcIndicesHandler
from src.core.handlers.step8_ml_train import Step8MlTrainHandler
from src.core.handlers.step9_ml_predict import Step9MlPredictHandler
from src.core.handlers.step10_qaa_inversion import Step10QaaInversionHandler
from src.core.handlers.step10_watercolor_handler import Step10WatercolorHandler
from src.core.handlers.step11_concentration import Step11ConcentrationHandler
from src.core.handlers.step11_map_handler import Step11MapHandler
from src.core.handlers.step12_kriging import Step12KrigingHandler
from src.core.handlers.step13_visualization import Step13VisualizationHandler
from src.core.handlers.step14_report import Step14ReportHandler
@ -51,6 +53,8 @@ def register_all_handlers(scheduler: PipelineScheduler):
scheduler.register_handler(Step8MlTrainHandler())
scheduler.register_handler(Step9MlPredictHandler())
scheduler.register_handler(Step10QaaInversionHandler())
scheduler.register_handler(Step10WatercolorHandler())
scheduler.register_handler(Step11MapHandler())
scheduler.register_handler(Step11ConcentrationHandler())
scheduler.register_handler(Step12KrigingHandler())
scheduler.register_handler(Step13VisualizationHandler())

41
src/core/steps/glint_removal_step.py
View File

@ -166,20 +166,41 @@ class GlintRemovalStep:
notify("skipped", "跳过去耀斑，使用原始影像")
return img_path
# ---- 确定水域掩膜 ----
# ---- 确定水域掩膜（★ 防护临时文件竞态） ----
final_water_mask = water_mask
if final_water_mask is not None and str(final_water_mask).lower().endswith(".shp"):
if final_water_mask is not None and isinstance(final_water_mask, str):
fp = str(final_water_mask)
# 过滤临时文件后缀：__tmp / _tmp / _tmp_delete 不应作为输入路径
import re
_tmp_re = re.compile(r'(__tmp|_tmp_delete|_tmp)(?=\.\w+$|$)', re.IGNORECASE)
if _tmp_re.search(os.path.basename(fp)):
stable = _tmp_re.sub('', os.path.basename(fp))
stable = re.sub(r'_{2,}', '_', stable) # 修复双下划线残留
alt = os.path.join(os.path.dirname(fp), stable)
if os.path.isfile(alt):
print(f" [glint_removal] 检测到临时文件路径，自动替换为稳定文件: "
f"{os.path.basename(fp)} → {stable}")
final_water_mask = alt
else:
print(f" [glint_removal] 警告: 输入掩膜路径疑似临时文件，"
f"但稳定文件不存在: {stable}")
# shp 自动替换为 dat
dat_mask = str(Path(water_mask_dir) / "water_mask_from_shp.dat")
if Path(dat_mask).exists():
print(f"检测到输入掩膜为 .shp，自动替换为栅格掩膜: {dat_mask}")
final_water_mask = dat_mask
if final_water_mask is not None and str(final_water_mask).lower().endswith(".shp"):
dat_mask = str(Path(water_mask_dir) / "water_mask_from_shp.dat")
if Path(dat_mask).exists():
print(f"检测到输入掩膜为 .shp，自动替换为栅格掩膜: {dat_mask}")
final_water_mask = dat_mask
if final_water_mask is None:
dat_mask_default = str(Path(water_mask_dir) / "water_mask_from_shp.dat")
if Path(dat_mask_default).exists():
final_water_mask = dat_mask_default
print(f"使用步骤1生成的水域掩膜: {final_water_mask}")
# ★ 按优先级查找稳定的掩膜文件（跳过临时文件）
for candidate_name in ("water_mask_from_shp.dat", "water_mask_out.dat",
"water_mask_from_ndwi.dat"):
candidate = str(Path(water_mask_dir) / candidate_name)
if Path(candidate).exists():
final_water_mask = candidate
print(f"使用步骤1生成的水域掩膜: {final_water_mask}")
break
# ---- 步骤3.1: 0值像素插值 ----
if interpolate_zeros:

88
src/core/utils/mask_converter.py
View File

@ -6,6 +6,8 @@
以及水体掩膜的预处理逻辑。
"""
import os
import re
import time
from pathlib import Path
from typing import Optional, Union
@ -61,6 +63,9 @@ def prepare_water_mask_for_algorithm(
# 字符串路径
if isinstance(water_mask, str):
# ★ 入口防御：自动将临时文件路径修正为稳定文件路径
water_mask = _resolve_stable_mask_path(water_mask)
ext = Path(water_mask).suffix.lower()
# shapefile 格式
@ -113,24 +118,83 @@ def _convert_shp_to_mask(shp_path: str, img_path: str,
return _load_raster_mask(temp_mask_path, image_shape[0], image_shape[1])
def _load_raster_mask(mask_path: str, img_height: int, img_width: int) -> np.ndarray:
"""从栅格文件加载掩膜"""
# ── 临时文件关键词（与 workspace_manager.TMP_KEYWORDS 保持一致）──
_TMP_PATTERN = re.compile(r'(__tmp|_tmp_delete|_tmp)(?=\.\w+$|$)', re.IGNORECASE)
def _resolve_stable_mask_path(mask_path: str) -> str:
"""如果 mask_path 指向临时文件，尝试解析为对应的稳定文件。
例如: "water_mask_out__tmp_delete.dat" → "water_mask_out.dat"
"""
if not _TMP_PATTERN.search(os.path.basename(mask_path)):
return mask_path
stable = _TMP_PATTERN.sub('', os.path.basename(mask_path))
stable_path = os.path.join(os.path.dirname(mask_path), stable)
if os.path.isfile(stable_path):
print(f" [mask_converter] 自动将临时路径解析为稳定文件: "
f"{os.path.basename(mask_path)} → {stable}")
return stable_path
# 尝试去除扩展名中多余的重复段（如 _tmp_delete 前后的双下划线残留）
stable2 = re.sub(r'_{2,}', '_', stable)
stable2_path = os.path.join(os.path.dirname(mask_path), stable2)
if os.path.isfile(stable2_path):
print(f" [mask_converter] 自动将临时路径解析为稳定文件: "
f"{os.path.basename(mask_path)} → {stable2}")
return stable2_path
return mask_path
def _load_raster_mask(mask_path: str, img_height: int, img_width: int,
retry_count: int = 3, retry_delay: float = 0.5) -> np.ndarray:
"""从栅格文件加载掩膜（带临时文件自动修复 + 重试机制）。
2026-06-30 修复:
- 自动将 __tmp_delete / _tmp 类临时路径解析为对应的稳定文件名
- 若文件不存在，短暂等待后重试（应对并发写入/删除的 TOCTOU 竞态）
"""
if not GDAL_AVAILABLE:
raise ImportError("GDAL未安装，无法读取掩膜文件")
mask_dataset = gdal.Open(mask_path, gdal.GA_ReadOnly)
if mask_dataset is None:
raise ValueError(f"无法打开掩膜文件: {mask_path}")
# ★ 临时文件 → 稳定文件自动修正
mask_path = _resolve_stable_mask_path(mask_path)
try:
mask_array = mask_dataset.GetRasterBand(1).ReadAsArray()
finally:
mask_dataset = None
last_error = None
for attempt in range(1, retry_count + 1):
if not os.path.isfile(mask_path):
last_error = FileNotFoundError(
f"掩膜文件不存在: {mask_path}"
)
if attempt < retry_count:
time.sleep(retry_delay)
continue
raise last_error
if mask_array.shape != (img_height, img_width):
raise ValueError(f"掩膜尺寸 {mask_array.shape} 与图像尺寸 {(img_height, img_width)} 不匹配")
mask_dataset = gdal.Open(mask_path, gdal.GA_ReadOnly)
if mask_dataset is None:
last_error = ValueError(f"无法打开掩膜文件: {mask_path}")
if attempt < retry_count:
time.sleep(retry_delay)
continue
raise last_error
return (mask_array > 0).astype(np.uint8)
try:
mask_array = mask_dataset.GetRasterBand(1).ReadAsArray()
finally:
mask_dataset = None
if mask_array.shape != (img_height, img_width):
raise ValueError(
f"掩膜尺寸 {mask_array.shape} 与图像尺寸 {(img_height, img_width)} 不匹配"
)
return (mask_array > 0).astype(np.uint8)
# 不应到达这里，但保持类型安全
raise last_error
def ensure_water_mask_dat(img_path: str,

2
src/core/visualization/scatter_plot.py
View File

@ -51,7 +51,7 @@ def generate_model_scatter_plots(
# 确定输出目录
if output_dir is None:
output_dir = str(Path(models_dir).parent / "14_visualization" / "scatter_plots")
output_dir = str(Path(models_dir).parent / "12_visualization" / "scatter_plots")
Path(output_dir).mkdir(parents=True, exist_ok=True)
# 实例化可视化器

14
src/core/workspace_manager.py
View File

@ -41,8 +41,8 @@ class WorkspaceManager:
'step7_index': {'training_spectra_indices': "7_Water_Quality_Indices/training_spectra_indices.csv"},
'step8_ml_train': {'Supervised_Model_Training': "8_Supervised_Model_Training/"},
'step9_ml_predict': {'9_ML_Prediction': "9_ML_Prediction/"},
'step10_watercolor': {'WaterIndex_Images': "10_WaterIndex_Images/"},
'step11_map': {'14_visualization': "14_visualization/"},
'step10_watercolor': {'WaterIndex_CSV': "10_WaterIndex_CSV/"},
'step11_map': {'11_Thematic_Map': "11_Thematic_Map/"},
}
self.step_outputs = {}
@ -159,17 +159,15 @@ class WorkspaceManager:
'1_water_mask': 'step1',
'2_Glint_Detection': 'step2',
'3_deglint': 'step3',
'4_sampling': 'step4_sampling',
'5_Data_Cleaning': 'step5_clean',
'6_Spectral_Feature_Extraction': 'step6_feature',
'7_Water_Quality_Indices': 'step7_index',
'8_Supervised_Model_Training': 'step8_ml_train',
'8_Regression_Modeling': 'step8_ml_train',
'13_Custom_Regression': 'step13',
'9_ML_Prediction': 'step9_ml_predict',
'11_12_13_predictions/Non_Empirical_Prediction': 'step11_map',
'13_Custom_Regression/Custom_Regression_Prediction': 'step13',
'14_visualization': 'step13_report',
'10_geotiff_batch_rendering': 'step11_map'
'10_WaterIndex_CSV': 'step10_watercolor',
'11_Thematic_Map': 'step11_map',
'12_visualization': ['step12_viz', 'step13_report'],
}
for subdir, step_ids in subdirs.items():