refactor(step10): 拆分 WaterIndexCsvProcessor 到独立子模块 + smoke test
This commit is contained in:
180
_smoke_test_step10.py
Normal file
180
_smoke_test_step10.py
Normal file
@ -0,0 +1,180 @@
|
|||||||
|
"""
|
||||||
|
Smoke test for Step 10 散点 CSV 模式 (WaterIndexCsvProcessor)
|
||||||
|
|
||||||
|
模拟 Step 4 输出格式 (sampling_spectra.csv):
|
||||||
|
x_coord, y_coord, pixel_x, pixel_y, "400.000000", "401.000000", ...
|
||||||
|
|
||||||
|
验证 WaterIndexCsvProcessor.compute_indices_from_csv:
|
||||||
|
1. 正确读取 x_coord/y_coord → 重命名为 longitude/latitude
|
||||||
|
2. 正确识别数字列名 = 光谱列
|
||||||
|
3. 复用 WaterQualityIndexCalculator 逐行计算
|
||||||
|
4. 输出每个公式一个 CSV,三列严格为 longitude, latitude, <formula_name>
|
||||||
|
5. 公式值数量级合理 (非 NaN,非 inf)
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# 让脚本能找到项目根
|
||||||
|
PROJECT_ROOT = Path(__file__).parent
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
|
||||||
|
def create_synthetic_sampling_csv(path: str, n_points: int = 5):
|
||||||
|
"""模拟 Step 4 输出: x_coord, y_coord, pixel_x, pixel_y, 数字列名光谱"""
|
||||||
|
import csv
|
||||||
|
# 选一组关键波段(确保 waterindex.csv 中的 BGA_Am09KBBI 等公式都能找到)
|
||||||
|
wavelengths = [400.0, 443.0, 458.0, 486.0, 500.0, 510.0, 531.0, 547.0, 555.0,
|
||||||
|
615.0, 622.0, 629.0, 644.0, 658.0, 665.0, 672.0, 681.0, 686.0,
|
||||||
|
700.0, 709.0, 714.0, 715.0, 753.0, 857.0, 900.0]
|
||||||
|
fieldnames = ['x_coord', 'y_coord', 'pixel_x', 'pixel_y'] + [f'{w:.6f}' for w in wavelengths]
|
||||||
|
|
||||||
|
with open(path, 'w', newline='', encoding='utf-8-sig') as f:
|
||||||
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||||
|
writer.writeheader()
|
||||||
|
for i in range(n_points):
|
||||||
|
# 模拟水体光谱(典型内陆湖泊反射率 0.005-0.05)
|
||||||
|
row = {
|
||||||
|
'x_coord': 100.0 + i * 10,
|
||||||
|
'y_coord': 30.0 + i * 5,
|
||||||
|
'pixel_x': 100 + i,
|
||||||
|
'pixel_y': 30 + i,
|
||||||
|
}
|
||||||
|
for w in wavelengths:
|
||||||
|
# 简单合成光谱: 蓝光 < 红光 + 一点叶绿素峰
|
||||||
|
base = 0.01 + 0.0001 * (w - 400)
|
||||||
|
chl_peak = 0.005 * (1 - abs(w - 560) / 200) if abs(w - 560) < 200 else 0
|
||||||
|
row[f'{w:.6f}'] = round(base + chl_peak, 6)
|
||||||
|
writer.writerow(row)
|
||||||
|
|
||||||
|
|
||||||
|
def run_smoke():
|
||||||
|
print("=" * 70)
|
||||||
|
print("Step 10 散点 CSV 模式 Smoke Test")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
tmpdir = tempfile.mkdtemp(prefix="step10_smoke_")
|
||||||
|
print(f"Tempdir: {tmpdir}")
|
||||||
|
|
||||||
|
sampling_csv = os.path.join(tmpdir, "sampling_spectra.csv")
|
||||||
|
output_dir = os.path.join(tmpdir, "10_WaterIndex_CSV")
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# 1) 创建合成 sampling CSV
|
||||||
|
create_synthetic_sampling_csv(sampling_csv, n_points=5)
|
||||||
|
print(f"Created sampling CSV: {sampling_csv}")
|
||||||
|
with open(sampling_csv, encoding='utf-8-sig') as f:
|
||||||
|
header_line = f.readline().strip()
|
||||||
|
print(f" header: {header_line[:120]}...")
|
||||||
|
|
||||||
|
# 2) 找到项目自带的 waterindex.csv
|
||||||
|
waterindex_csv = PROJECT_ROOT / "src" / "gui" / "model" / "waterindex.csv"
|
||||||
|
print(f"Using waterindex.csv: {waterindex_csv}")
|
||||||
|
assert waterindex_csv.is_file(), "waterindex.csv not found!"
|
||||||
|
|
||||||
|
# 3) 调 WaterIndexCsvProcessor
|
||||||
|
# 注意:父包 __init__.py 顶部有 `from osgeo import gdal, osr`,
|
||||||
|
# 在没装 gdal 的 venv 里任何 from ...waterindex_inversion import ... 都会炸。
|
||||||
|
# 这里用 importlib 按文件路径直接加载 csv_processor.py 子模块,
|
||||||
|
# 完全绕开 __init__.py 的 osgeo 加载。
|
||||||
|
import importlib.util as _ilu
|
||||||
|
_csv_proc_path = (
|
||||||
|
PROJECT_ROOT / "src" / "core" / "algorithms" / "waterindex_inversion"
|
||||||
|
/ "csv_processor.py"
|
||||||
|
)
|
||||||
|
_spec = _ilu.spec_from_file_location("waterindex_csv_processor", _csv_proc_path)
|
||||||
|
_mod = _ilu.module_from_spec(_spec)
|
||||||
|
sys.modules["waterindex_csv_processor"] = _mod
|
||||||
|
_spec.loader.exec_module(_mod)
|
||||||
|
WaterIndexCsvProcessor = _mod.WaterIndexCsvProcessor
|
||||||
|
|
||||||
|
progress_log = []
|
||||||
|
def progress_cb(msg, pct):
|
||||||
|
progress_log.append((msg, pct))
|
||||||
|
|
||||||
|
proc = WaterIndexCsvProcessor(str(waterindex_csv))
|
||||||
|
print(f"\n[Step] compute_indices_from_csv...")
|
||||||
|
out_files = proc.compute_indices_from_csv(
|
||||||
|
sampling_csv_path=sampling_csv,
|
||||||
|
output_dir=output_dir,
|
||||||
|
selected_formulas=["BGA_Am09KBBI", "BGA_Da052BDA", "BGA_Be16NDPhyI"],
|
||||||
|
progress_callback=progress_cb,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"\n[Result] Generated {len(out_files)} CSV files:")
|
||||||
|
for name, path in out_files.items():
|
||||||
|
size = os.path.getsize(path)
|
||||||
|
print(f" {name:30s} -> {os.path.basename(path)} ({size} bytes)")
|
||||||
|
|
||||||
|
# 4) 验证每个输出 CSV 的列结构
|
||||||
|
print(f"\n[Verify] Column structure check:")
|
||||||
|
all_pass = True
|
||||||
|
import pandas as pd
|
||||||
|
for name, path in out_files.items():
|
||||||
|
df = pd.read_csv(path, encoding='utf-8-sig')
|
||||||
|
cols = list(df.columns)
|
||||||
|
expected = ['longitude', 'latitude', name]
|
||||||
|
ok = (cols == expected) and (len(df) == 5)
|
||||||
|
flag = "✓" if ok else "✗"
|
||||||
|
if not ok:
|
||||||
|
all_pass = False
|
||||||
|
print(f" {flag} {name:30s} cols={cols} rows={len(df)}")
|
||||||
|
|
||||||
|
# 5) 验证坐标重命名
|
||||||
|
print(f"\n[Verify] Coordinate rename (x_coord→longitude, y_coord→latitude):")
|
||||||
|
sample = pd.read_csv(out_files[list(out_files.keys())[0]], encoding='utf-8-sig')
|
||||||
|
print(f" longitude values: {sample['longitude'].tolist()}")
|
||||||
|
print(f" latitude values: {sample['latitude'].tolist()}")
|
||||||
|
coord_ok = (sample['longitude'].iloc[0] == 100.0 and
|
||||||
|
sample['latitude'].iloc[0] == 30.0)
|
||||||
|
if not coord_ok:
|
||||||
|
all_pass = False
|
||||||
|
print(f" {'✓' if coord_ok else '✗'} coordinate rename correct")
|
||||||
|
|
||||||
|
# 6) 验证公式值非 NaN
|
||||||
|
print(f"\n[Verify] Formula values (no NaN):")
|
||||||
|
for name, path in out_files.items():
|
||||||
|
df = pd.read_csv(path, encoding='utf-8-sig')
|
||||||
|
col = df[name]
|
||||||
|
n_nan = col.isna().sum()
|
||||||
|
n_inf = ((col == float('inf')) | (col == float('-inf'))).sum()
|
||||||
|
all_nan = col.dropna().empty
|
||||||
|
if n_nan > 0 or n_inf > 0 or all_nan:
|
||||||
|
print(f" ✗ {name:30s}: NaN={n_nan} Inf={n_inf} empty={all_nan}")
|
||||||
|
print(f" values: {col.tolist()}")
|
||||||
|
all_pass = False
|
||||||
|
else:
|
||||||
|
mn, mx = col.min(), col.max()
|
||||||
|
print(f" ✓ {name:30s}: range=[{mn:.4f}, {mx:.4f}]")
|
||||||
|
|
||||||
|
# 7) 进度回调检查
|
||||||
|
print(f"\n[Verify] Progress callback:")
|
||||||
|
print(f" Total progress events: {len(progress_log)}")
|
||||||
|
if progress_log:
|
||||||
|
first_msg, first_pct = progress_log[0]
|
||||||
|
last_msg, last_pct = progress_log[-1]
|
||||||
|
print(f" First: ({first_pct:.1f}%) {first_msg}")
|
||||||
|
print(f" Last : ({last_pct:.1f}%) {last_msg}")
|
||||||
|
progress_ok = (last_pct == 100.0)
|
||||||
|
if not progress_ok:
|
||||||
|
all_pass = False
|
||||||
|
print(f" {'✓' if progress_ok else '✗'} last progress = 100%")
|
||||||
|
|
||||||
|
# 8) 总结
|
||||||
|
print(f"\n{'=' * 70}")
|
||||||
|
if all_pass:
|
||||||
|
print(f"✓ ALL CHECKS PASSED")
|
||||||
|
else:
|
||||||
|
print(f"✗ SOME CHECKS FAILED — inspect output above")
|
||||||
|
print(f"{'=' * 70}")
|
||||||
|
|
||||||
|
# 清理
|
||||||
|
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||||
|
return all_pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
success = run_smoke()
|
||||||
|
sys.exit(0 if success else 1)
|
||||||
@ -649,184 +649,11 @@ class WaterIndexProcessor:
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# 散点处理入口(Step 10 重构后使用,与 Step 9 对称)
|
# 散点处理入口(Step 10 重构后使用,与 Step 9 对称)
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
# WaterIndexCsvProcessor 已拆出到独立子模块 csv_processor.py,
|
||||||
class WaterIndexCsvProcessor:
|
# 目的是让纯 CSV 计算链路不再被 __init__.py 顶部 osgeo import 拖垮。
|
||||||
"""
|
# 这里做一次 re-export,保留所有 `from src.core.algorithms.waterindex_inversion import WaterIndexCsvProcessor`
|
||||||
散点 CSV 驱动的水色指数反演器。
|
# 这类已有 import 路径仍能正常工作(生产环境/打包后)。
|
||||||
|
from src.core.algorithms.waterindex_inversion.csv_processor import WaterIndexCsvProcessor # noqa: E402,F401
|
||||||
设计目的
|
|
||||||
--------
|
|
||||||
与 Step 9 (ML 预测) 完全对称的【散点处理模式】:
|
|
||||||
|
|
||||||
* 输入:Step 4 生成的 ``sampling_spectra.csv``,列结构为
|
|
||||||
``x_coord, y_coord, pixel_x, pixel_y, 400.000000, 401.000000, ...``
|
|
||||||
* 处理:解析 ``waterindex.csv`` 中的公式,对每行采样点
|
|
||||||
提取对应波段数值、逐行 eval 计算水色指数
|
|
||||||
* 输出:每个公式一个 CSV,列严格为 ``longitude, latitude, <formula_name>``,
|
|
||||||
可直接喂给 Step 11 ContentMapper
|
|
||||||
|
|
||||||
输出目录
|
|
||||||
--------
|
|
||||||
默认 ``{work_dir}/10_WaterIndex_CSV/``;若用户指定 ``output_dir`` 则用其值。
|
|
||||||
"""
|
|
||||||
|
|
||||||
COORD_RENAME_MAP = {
|
|
||||||
"x_coord": "longitude",
|
|
||||||
"y_coord": "latitude",
|
|
||||||
"lon": "longitude",
|
|
||||||
"lat": "latitude",
|
|
||||||
}
|
|
||||||
|
|
||||||
def __init__(self, waterindex_csv_path: Optional[str] = None):
|
|
||||||
if waterindex_csv_path is None:
|
|
||||||
candidates = [
|
|
||||||
os.path.join(os.path.dirname(__file__), '..', '..', 'gui', 'model', 'waterindex.csv'),
|
|
||||||
os.path.join(os.path.dirname(__file__), '..', '..', '..', 'gui', 'model', 'waterindex.csv'),
|
|
||||||
]
|
|
||||||
for p in candidates:
|
|
||||||
if os.path.isfile(p):
|
|
||||||
waterindex_csv_path = p
|
|
||||||
break
|
|
||||||
self.waterindex_csv_path = waterindex_csv_path
|
|
||||||
self._index_calc = None
|
|
||||||
|
|
||||||
def _get_index_calc(self):
|
|
||||||
"""懒加载 WaterQualityIndexCalculator(首次访问时实例化)"""
|
|
||||||
if self._index_calc is None and self.waterindex_csv_path:
|
|
||||||
from src.utils.water_index import WaterQualityIndexCalculator
|
|
||||||
self._index_calc = WaterQualityIndexCalculator(self.waterindex_csv_path)
|
|
||||||
return self._index_calc
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _detect_wavelength_columns(df: "pd.DataFrame") -> List[str]:
|
|
||||||
"""识别光谱列:列名是浮点数字符串(Step 4 输出的 '400.000000' 形式)"""
|
|
||||||
import re
|
|
||||||
wl_cols = []
|
|
||||||
for col in df.columns:
|
|
||||||
try:
|
|
||||||
float(str(col).strip())
|
|
||||||
wl_cols.append(col)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
continue
|
|
||||||
return wl_cols
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _safe_filename(name: str) -> str:
|
|
||||||
"""公式名 → 文件名安全字符(与旧 BSQ 输出命名习惯一致)"""
|
|
||||||
return re.sub(r'[^\w\u4e00-\u9fff-]', '_', name).strip('_') or 'index'
|
|
||||||
|
|
||||||
def compute_indices_from_csv(
|
|
||||||
self,
|
|
||||||
sampling_csv_path: str,
|
|
||||||
output_dir: str,
|
|
||||||
selected_formulas: Optional[List[str]] = None,
|
|
||||||
progress_callback: Optional[Callable[[str, float], None]] = None,
|
|
||||||
) -> Dict[str, str]:
|
|
||||||
"""
|
|
||||||
散点 CSV → 按指数拆分的多个 CSV。
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
sampling_csv_path : str
|
|
||||||
Step 4 输出的 ``sampling_spectra.csv`` 路径
|
|
||||||
output_dir : str
|
|
||||||
输出目录;不存在会自动创建
|
|
||||||
selected_formulas : list, optional
|
|
||||||
要计算的公式名列表;None 或空列表 = 全部公式
|
|
||||||
progress_callback : callable, optional
|
|
||||||
进度回调 ``(msg: str, pct: float)``
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
dict
|
|
||||||
``{公式名: 输出 CSV 路径}``
|
|
||||||
"""
|
|
||||||
def notify(msg: str, pct: float) -> None:
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(msg, pct)
|
|
||||||
|
|
||||||
if not os.path.isfile(sampling_csv_path):
|
|
||||||
raise FileNotFoundError(f"采样点 CSV 不存在: {sampling_csv_path}")
|
|
||||||
|
|
||||||
if not self.waterindex_csv_path or not os.path.isfile(self.waterindex_csv_path):
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"waterindex.csv 未配置或不存在: {self.waterindex_csv_path}"
|
|
||||||
)
|
|
||||||
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
|
|
||||||
notify("正在读取采样点 CSV…", 5)
|
|
||||||
import pandas as pd
|
|
||||||
df = pd.read_csv(sampling_csv_path, encoding="utf-8-sig")
|
|
||||||
if df.empty:
|
|
||||||
raise ValueError(f"采样点 CSV 为空: {sampling_csv_path}")
|
|
||||||
|
|
||||||
# 坐标列重命名(x_coord → longitude, y_coord → latitude)
|
|
||||||
df = df.rename(columns={k: v for k, v in self.COORD_RENAME_MAP.items()
|
|
||||||
if k in df.columns})
|
|
||||||
if "longitude" not in df.columns or "latitude" not in df.columns:
|
|
||||||
raise ValueError(
|
|
||||||
f"采样点 CSV 缺少坐标列(期望 x_coord/y_coord 或 longitude/latitude),"
|
|
||||||
f"实际列: {list(df.columns)}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 识别光谱列
|
|
||||||
wl_cols = self._detect_wavelength_columns(df)
|
|
||||||
if not wl_cols:
|
|
||||||
raise ValueError(
|
|
||||||
f"采样点 CSV 中未识别到任何光谱列(列名为数字),"
|
|
||||||
f"实际列: {list(df.columns)}"
|
|
||||||
)
|
|
||||||
notify(f"识别到 {len(wl_cols)} 个光谱列, 采样点 {len(df)} 个", 15)
|
|
||||||
|
|
||||||
calc = self._get_index_calc()
|
|
||||||
if calc is None:
|
|
||||||
raise RuntimeError("WaterQualityIndexCalculator 初始化失败")
|
|
||||||
|
|
||||||
all_formula_names = calc.list_available()
|
|
||||||
if selected_formulas:
|
|
||||||
targets = [n for n in selected_formulas if n in all_formula_names]
|
|
||||||
missing = [n for n in selected_formulas if n not in all_formula_names]
|
|
||||||
if missing:
|
|
||||||
print(f"[WaterIndexCsvProcessor] 警告: 以下公式未在 waterindex.csv 中找到,已跳过: {missing}")
|
|
||||||
else:
|
|
||||||
targets = all_formula_names
|
|
||||||
|
|
||||||
if not targets:
|
|
||||||
raise ValueError("没有可计算的公式(selected_formulas 为空且 waterindex.csv 中无公式)")
|
|
||||||
|
|
||||||
# 一次性算出所有目标公式的 Series(避免重复遍历 DataFrame)
|
|
||||||
notify(f"开始逐行计算 {len(targets)} 个公式…", 25)
|
|
||||||
spectra_df = df[wl_cols]
|
|
||||||
try:
|
|
||||||
results_df = calc.calculate_many(targets, spectra_df)
|
|
||||||
except Exception as e:
|
|
||||||
raise RuntimeError(f"公式计算失败: {e}")
|
|
||||||
|
|
||||||
# 每个公式一个 CSV:longitude, latitude, <formula_name>
|
|
||||||
out_files: Dict[str, str] = {}
|
|
||||||
n_total = len(targets)
|
|
||||||
for i, name in enumerate(targets):
|
|
||||||
try:
|
|
||||||
per_idx = results_df[name]
|
|
||||||
out_df = pd.DataFrame({
|
|
||||||
"longitude": df["longitude"].values,
|
|
||||||
"latitude": df["latitude"].values,
|
|
||||||
name: per_idx.values,
|
|
||||||
})
|
|
||||||
out_path = os.path.join(output_dir, f"{self._safe_filename(name)}.csv")
|
|
||||||
out_df.to_csv(out_path, index=False, float_format="%.6f", encoding="utf-8-sig")
|
|
||||||
out_files[name] = out_path
|
|
||||||
notify(
|
|
||||||
f"[{i + 1}/{n_total}] {name} → {os.path.basename(out_path)}",
|
|
||||||
25 + 70 * (i + 1) / n_total,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[WaterIndexCsvProcessor] 公式 '{name}' 失败: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
notify(f"完成!共输出 {len(out_files)} / {n_total} 个指数 CSV", 100)
|
|
||||||
return out_files
|
|
||||||
|
|
||||||
|
|
||||||
# 保留旧 import 路径兼容
|
# 保留旧 import 路径兼容
|
||||||
|
|||||||
220
src/core/algorithms/waterindex_inversion/csv_processor.py
Normal file
220
src/core/algorithms/waterindex_inversion/csv_processor.py
Normal file
@ -0,0 +1,220 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
水色指数反演 — 散点 CSV 模式处理器(独立子模块)。
|
||||||
|
|
||||||
|
设计意图
|
||||||
|
--------
|
||||||
|
本模块与 ``waterindex_inversion.__init__.py`` 中的 ``WaterIndexProcessor``
|
||||||
|
(栅格 BSQ 模式) **彻底解耦**,不依赖任何 osgeo / rasterio / gdal,仅依赖
|
||||||
|
``pandas`` 与 ``src.utils.water_index.WaterQualityIndexCalculator``。
|
||||||
|
|
||||||
|
**为什么独立成文件?**
|
||||||
|
|
||||||
|
``__init__.py`` 顶部有 ``from osgeo import gdal, osr``(用于 BSQ 栅格模式),
|
||||||
|
这意味着任何 ``from src.core.algorithms.waterindex_inversion import X``
|
||||||
|
都会触发 osgeo 加载——而某些验证环境(无 gdal 包的 venv)会因此 ImportError。
|
||||||
|
|
||||||
|
本子模块独立后,可通过
|
||||||
|
``from src.core.algorithms.waterindex_inversion.csv_processor import WaterIndexCsvProcessor``
|
||||||
|
直接加载,**完全不触发** ``__init__.py`` 的 osgeo import 链,便于无 gdal 环境
|
||||||
|
做端到端 smoke test。
|
||||||
|
|
||||||
|
**调用入口(由 Step 10 service / panel 调用)**::
|
||||||
|
|
||||||
|
from src.core.algorithms.waterindex_inversion.csv_processor import WaterIndexCsvProcessor
|
||||||
|
proc = WaterIndexCsvProcessor(waterindex_csv_path)
|
||||||
|
out = proc.compute_indices_from_csv(
|
||||||
|
sampling_csv_path=...,
|
||||||
|
output_dir=...,
|
||||||
|
selected_formulas=[...],
|
||||||
|
progress_callback=lambda msg, pct: ...,
|
||||||
|
)
|
||||||
|
|
||||||
|
输出格式
|
||||||
|
--------
|
||||||
|
每个公式一个 CSV,三列严格为 ``longitude, latitude, <formula_name>``。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
class WaterIndexCsvProcessor:
|
||||||
|
"""
|
||||||
|
散点 CSV 驱动的水色指数反演器。
|
||||||
|
|
||||||
|
设计目的
|
||||||
|
--------
|
||||||
|
与 Step 9 (ML 预测) 完全对称的【散点处理模式】:
|
||||||
|
|
||||||
|
* 输入:Step 4 生成的 ``sampling_spectra.csv``,列结构为
|
||||||
|
``x_coord, y_coord, pixel_x, pixel_y, 400.000000, 401.000000, ...``
|
||||||
|
* 处理:解析 ``waterindex.csv`` 中的公式,对每行采样点
|
||||||
|
提取对应波段数值、逐行 eval 计算水色指数
|
||||||
|
* 输出:每个公式一个 CSV,列严格为 ``longitude, latitude, <formula_name>``,
|
||||||
|
可直接喂给 Step 11 ContentMapper
|
||||||
|
|
||||||
|
输出目录
|
||||||
|
--------
|
||||||
|
默认 ``{work_dir}/10_WaterIndex_CSV/``;若用户指定 ``output_dir`` 则用其值。
|
||||||
|
"""
|
||||||
|
|
||||||
|
COORD_RENAME_MAP = {
|
||||||
|
"x_coord": "longitude",
|
||||||
|
"y_coord": "latitude",
|
||||||
|
"lon": "longitude",
|
||||||
|
"lat": "latitude",
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, waterindex_csv_path: Optional[str] = None):
|
||||||
|
if waterindex_csv_path is None:
|
||||||
|
candidates = [
|
||||||
|
os.path.join(os.path.dirname(__file__), '..', '..', 'gui', 'model', 'waterindex.csv'),
|
||||||
|
os.path.join(os.path.dirname(__file__), '..', '..', '..', 'gui', 'model', 'waterindex.csv'),
|
||||||
|
]
|
||||||
|
for p in candidates:
|
||||||
|
if os.path.isfile(p):
|
||||||
|
waterindex_csv_path = p
|
||||||
|
break
|
||||||
|
self.waterindex_csv_path = waterindex_csv_path
|
||||||
|
self._index_calc = None
|
||||||
|
|
||||||
|
def _get_index_calc(self):
|
||||||
|
"""懒加载 WaterQualityIndexCalculator(首次访问时实例化)"""
|
||||||
|
if self._index_calc is None and self.waterindex_csv_path:
|
||||||
|
from src.utils.water_index import WaterQualityIndexCalculator
|
||||||
|
self._index_calc = WaterQualityIndexCalculator(self.waterindex_csv_path)
|
||||||
|
return self._index_calc
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _detect_wavelength_columns(df: "object") -> List[str]:
|
||||||
|
"""识别光谱列:列名是浮点数字符串(Step 4 输出的 '400.000000' 形式)"""
|
||||||
|
wl_cols = []
|
||||||
|
for col in df.columns:
|
||||||
|
try:
|
||||||
|
float(str(col).strip())
|
||||||
|
wl_cols.append(col)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
return wl_cols
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _safe_filename(name: str) -> str:
|
||||||
|
"""公式名 → 文件名安全字符(与旧 BSQ 输出命名习惯一致)"""
|
||||||
|
return re.sub(r'[^\w\u4e00-\u9fff-]', '_', name).strip('_') or 'index'
|
||||||
|
|
||||||
|
def compute_indices_from_csv(
|
||||||
|
self,
|
||||||
|
sampling_csv_path: str,
|
||||||
|
output_dir: str,
|
||||||
|
selected_formulas: Optional[List[str]] = None,
|
||||||
|
progress_callback: Optional[Callable[[str, float], None]] = None,
|
||||||
|
) -> Dict[str, str]:
|
||||||
|
"""
|
||||||
|
散点 CSV → 按指数拆分的多个 CSV。
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
sampling_csv_path : str
|
||||||
|
Step 4 输出的 ``sampling_spectra.csv`` 路径
|
||||||
|
output_dir : str
|
||||||
|
输出目录;不存在会自动创建
|
||||||
|
selected_formulas : list, optional
|
||||||
|
要计算的公式名列表;None 或空列表 = 全部公式
|
||||||
|
progress_callback : callable, optional
|
||||||
|
进度回调 ``(msg: str, pct: float)``
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
dict
|
||||||
|
``{公式名: 输出 CSV 路径}``
|
||||||
|
"""
|
||||||
|
def notify(msg: str, pct: float) -> None:
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback(msg, pct)
|
||||||
|
|
||||||
|
if not os.path.isfile(sampling_csv_path):
|
||||||
|
raise FileNotFoundError(f"采样点 CSV 不存在: {sampling_csv_path}")
|
||||||
|
|
||||||
|
if not self.waterindex_csv_path or not os.path.isfile(self.waterindex_csv_path):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"waterindex.csv 未配置或不存在: {self.waterindex_csv_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
|
|
||||||
|
notify("正在读取采样点 CSV…", 5)
|
||||||
|
import pandas as pd
|
||||||
|
df = pd.read_csv(sampling_csv_path, encoding="utf-8-sig")
|
||||||
|
if df.empty:
|
||||||
|
raise ValueError(f"采样点 CSV 为空: {sampling_csv_path}")
|
||||||
|
|
||||||
|
# 坐标列重命名(x_coord → longitude, y_coord → latitude)
|
||||||
|
df = df.rename(columns={k: v for k, v in self.COORD_RENAME_MAP.items()
|
||||||
|
if k in df.columns})
|
||||||
|
if "longitude" not in df.columns or "latitude" not in df.columns:
|
||||||
|
raise ValueError(
|
||||||
|
f"采样点 CSV 缺少坐标列(期望 x_coord/y_coord 或 longitude/latitude),"
|
||||||
|
f"实际列: {list(df.columns)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 识别光谱列
|
||||||
|
wl_cols = self._detect_wavelength_columns(df)
|
||||||
|
if not wl_cols:
|
||||||
|
raise ValueError(
|
||||||
|
f"采样点 CSV 中未识别到任何光谱列(列名为数字),"
|
||||||
|
f"实际列: {list(df.columns)}"
|
||||||
|
)
|
||||||
|
notify(f"识别到 {len(wl_cols)} 个光谱列, 采样点 {len(df)} 个", 15)
|
||||||
|
|
||||||
|
calc = self._get_index_calc()
|
||||||
|
if calc is None:
|
||||||
|
raise RuntimeError("WaterQualityIndexCalculator 初始化失败")
|
||||||
|
|
||||||
|
all_formula_names = calc.list_available()
|
||||||
|
if selected_formulas:
|
||||||
|
targets = [n for n in selected_formulas if n in all_formula_names]
|
||||||
|
missing = [n for n in selected_formulas if n not in all_formula_names]
|
||||||
|
if missing:
|
||||||
|
print(f"[WaterIndexCsvProcessor] 警告: 以下公式未在 waterindex.csv 中找到,已跳过: {missing}")
|
||||||
|
else:
|
||||||
|
targets = all_formula_names
|
||||||
|
|
||||||
|
if not targets:
|
||||||
|
raise ValueError("没有可计算的公式(selected_formulas 为空且 waterindex.csv 中无公式)")
|
||||||
|
|
||||||
|
# 一次性算出所有目标公式的 Series(避免重复遍历 DataFrame)
|
||||||
|
notify(f"开始逐行计算 {len(targets)} 个公式…", 25)
|
||||||
|
spectra_df = df[wl_cols]
|
||||||
|
try:
|
||||||
|
results_df = calc.calculate_many(targets, spectra_df)
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError(f"公式计算失败: {e}")
|
||||||
|
|
||||||
|
# 每个公式一个 CSV:longitude, latitude, <formula_name>
|
||||||
|
out_files: Dict[str, str] = {}
|
||||||
|
n_total = len(targets)
|
||||||
|
for i, name in enumerate(targets):
|
||||||
|
try:
|
||||||
|
per_idx = results_df[name]
|
||||||
|
out_df = pd.DataFrame({
|
||||||
|
"longitude": df["longitude"].values,
|
||||||
|
"latitude": df["latitude"].values,
|
||||||
|
name: per_idx.values,
|
||||||
|
})
|
||||||
|
out_path = os.path.join(output_dir, f"{self._safe_filename(name)}.csv")
|
||||||
|
out_df.to_csv(out_path, index=False, float_format="%.6f", encoding="utf-8-sig")
|
||||||
|
out_files[name] = out_path
|
||||||
|
notify(
|
||||||
|
f"[{i + 1}/{n_total}] {name} → {os.path.basename(out_path)}",
|
||||||
|
25 + 70 * (i + 1) / n_total,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[WaterIndexCsvProcessor] 公式 '{name}' 失败: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
notify(f"完成!共输出 {len(out_files)} / {n_total} 个指数 CSV", 100)
|
||||||
|
return out_files
|
||||||
Reference in New Issue
Block a user