格式统一

This commit is contained in:
duxin
2026-07-01 11:32:32 +08:00
parent a3c20d3e49
commit 8de73db80e
5 changed files with 89 additions and 60 deletions

View File

@ -211,7 +211,9 @@ class WaterIndexCsvProcessor:
skipped_count += 1 skipped_count += 1
continue continue
try: try:
per_idx = results_df[name] # ★ .copy() 确保拿到独立内存,防止上游向量化 eval 的临时视图
# 在后续 .replace / .clip / .to_csv 中触发 0xC0000005
per_idx = results_df[name].copy()
# ===== P0 防御: 写盘前清洗(防 Step 11 Kriging / TIN 碎玻璃)===== # ===== P0 防御: 写盘前清洗(防 Step 11 Kriging / TIN 碎玻璃)=====
# 1) inf / -inf → NaNpandas 默认会把 inf 写成字面 "Infinity" # 1) inf / -inf → NaNpandas 默认会把 inf 写成字面 "Infinity"
# 下游 ContentMapper 严格按位置读第 3 列时会原样拿到 inf # 下游 ContentMapper 严格按位置读第 3 列时会原样拿到 inf

View File

@ -431,7 +431,8 @@ class WaterQualityInference:
if not formulas: if not formulas:
print("[特征补全] Calculator 未持有任何公式,跳过补全") print("[特征补全] Calculator 未持有任何公式,跳过补全")
else: else:
results_df = calc.calculate_many(formulas, spectra) # fast=True: Step9 特征补全走向量化快车道(~63 公式,已验证稳定)
results_df = calc.calculate_many(formulas, spectra, fast=True)
# results_df 是列对齐的 WQI 计算结果(每列一个公式,行数=样本数) # results_df 是列对齐的 WQI 计算结果(每列一个公式,行数=样本数)
if isinstance(results_df, pd.DataFrame) and not results_df.empty: if isinstance(results_df, pd.DataFrame) and not results_df.empty:
original_col_count = spectra.shape[1] original_col_count = spectra.shape[1]

View File

@ -687,8 +687,10 @@ class ImageCategoryTree(QTreeWidget):
chart_type = "光谱曲线图" chart_type = "光谱曲线图"
elif "HEATMAP" in name_upper or "热力图" in name_upper: elif "HEATMAP" in name_upper or "热力图" in name_upper:
chart_type = "相关性热力图" chart_type = "相关性热力图"
elif "BOXPLOT" in name_upper or "HISTOGRAM" in name_upper or "箱线" in name_upper or "直方" in name_upper: elif "BOXPLOT" in name_upper or "箱线" in name_upper:
chart_type = "统计箱线图" chart_type = "统计箱线图"
elif "HISTOGRAM" in name_upper or "直方" in name_upper:
chart_type = "分布直方图"
elif "SAMPLING" in name_upper or "采样" in name_upper: elif "SAMPLING" in name_upper or "采样" in name_upper:
chart_type = "采样点地图" chart_type = "采样点地图"
elif "GLINT" in name_upper or "MASK" in name_upper or "PREVIEW" in name_upper: elif "GLINT" in name_upper or "MASK" in name_upper or "PREVIEW" in name_upper:
@ -1531,7 +1533,7 @@ class Step12VizPanel(QWidget):
self.view_mode_cb.currentIndexChanged.connect(self.update_image_tree_view) self.view_mode_cb.currentIndexChanged.connect(self.update_image_tree_view)
self.chart_filter_cb = QComboBox() self.chart_filter_cb = QComboBox()
self.chart_filter_cb.addItems(["全部图表", "空间分布图", "模型散点图", "光谱曲线图", "统计箱线图", "相关性热力图", "掩膜与预览", "采样点地图"]) self.chart_filter_cb.addItems(["全部图表", "空间分布图", "模型散点图", "光谱曲线图", "统计箱线图", "分布直方图", "相关性热力图", "掩膜与预览", "采样点地图"])
self.chart_filter_cb.currentIndexChanged.connect(self.update_image_tree_view) self.chart_filter_cb.currentIndexChanged.connect(self.update_image_tree_view)
filter_layout.addRow("视图模式:", self.view_mode_cb) filter_layout.addRow("视图模式:", self.view_mode_cb)

View File

@ -585,7 +585,8 @@ class ContentMapper:
verbose=False, verbose=False,
enable_plotting=False enable_plotting=False
) )
z, _ = ok.execute('grid', grid_x, grid_y) # ★ 局部邻域 Kriging只参考最近的 15 个点,避免万阶矩阵求逆
z, _ = ok.execute('grid', grid_x, grid_y, backend='loop', n_closest_points=15)
grid_content = np.array(z) grid_content = np.array(z)
valid_count = np.sum(~np.isnan(grid_content)) valid_count = np.sum(~np.isnan(grid_content))
print(f"Kriging插值成功有效点数: {valid_count} / {grid_content.size}") print(f"Kriging插值成功有效点数: {valid_count} / {grid_content.size}")

View File

@ -6,6 +6,8 @@ import re
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional, Union from typing import Dict, List, Optional, Union
from .band_math import BandMathCalculator
def _get_resource_path(relative_path: str) -> str: def _get_resource_path(relative_path: str) -> str:
"""获取资源文件路径,兼容开发/PyInstaller onedir/onefile 三种环境。""" """获取资源文件路径,兼容开发/PyInstaller onedir/onefile 三种环境。"""
@ -88,62 +90,80 @@ class WaterQualityIndexCalculator:
parts = [float(x.strip()) for x in s.split(",")] parts = [float(x.strip()) for x in s.split(",")]
return np.array(parts) return np.array(parts)
def _band_math_all_rows(self, df: pd.DataFrame, expression: str, wavelength_offset: float = 0.0) -> pd.Series: def _band_math_all_rows(self, df: pd.DataFrame, expression: str, wavelength_offset: float = 0.0,
fast: bool = False) -> pd.Series:
""" """
向量化批量计算波段表达式2026-07-01 重写:逐行 eval → 全列 numpy 波段表达式批量计算
原先逐行 eval + re.sub 在 11310 行 × 63 公式时超过 600 秒; Args:
现在一次性解析表达式为 numpy 操作,全 DataFrame 向量化计算, fast=True → 优先向量化全列 numpy但极少数公式可能触发 0xC0000005
同等数据量下 < 1 秒。 fast=False → 纯逐行 eval稳定不会崩
""" """
# ── 1. 从列名提取波长列表 ── calc = BandMathCalculator.__new__(BandMathCalculator)
wavelengths = [] calc.df = df
for col in df.columns: calc.wavelengths = calc._extract_wavelengths()
nums = re.findall(r'\d+\.?\d*', str(col)) calc.wavelength_offset = float(wavelength_offset)
wavelengths.append(float(nums[0]) if nums else None)
# ── 2. 解析表达式中的变量 (wNNN / WNNN) → 找到对应列索引 ── variables = calc._parse_expression(expression)
var_pattern = r'[wW](\d+\.?\d*)'
var_matches = re.findall(var_pattern, expression) var_col_map = {}
col_map = {} # {原始变量文本: 列索引} for var_str in variables:
for var_str in var_matches: wavelength = float(var_str) + wavelength_offset
target_wl = float(var_str) + wavelength_offset valid = [(i, wl) for i, wl in enumerate(calc.wavelengths) if wl is not None]
valid = [(i, wl) for i, wl in enumerate(wavelengths) if wl is not None]
if not valid: if not valid:
raise ValueError("未找到有效的波长列") continue
best_idx = min(valid, key=lambda x: abs(x[1] - target_wl))[0] col_idx = min(valid, key=lambda x: abs(x[1] - wavelength))[0]
# 同时覆盖 wNNN 和 WNNN 两种写法 var_col_map[f'w{var_str}'] = col_idx
col_map[f'w{var_str}'] = best_idx var_col_map[f'W{var_str}'] = col_idx
col_map[f'W{var_str}'] = best_idx
# ── 3. 构建带缓存的向量化 numpy 表达式 ── if not var_col_map:
# 注意w715 这种变量在 Python 中是合法标识符,但 eval 中会当变量名; return pd.Series(np.nan, index=df.index, name=expression)
# 我们直接替换为 arr[:, col_idx] 再传给 eval确保一次性全列计算。
arr = df.values # (N, M) numpy array避免重复 .iloc 访问
eval_expr = expression
# 按变量名长度降序替换,防止短变量吞噬长变量前缀(如 w715 先于 w71
for var_name in sorted(col_map.keys(), key=len, reverse=True):
col_idx = col_map[var_name]
# 替换为安全的列引用,匹配完整 token用 \b 边界)
eval_expr = re.sub(
r'\b' + re.escape(var_name) + r'\b',
f'arr[:, {col_idx}]',
eval_expr,
)
# ── 4. 一次性向量化求值 ── # ── 快车道:向量化全列 numpy仅 fast=True 时启用)──
try: if fast:
with np.errstate(divide='ignore', invalid='ignore'): try:
result = eval(eval_expr, {"__builtins__": None}, {"arr": arr, "np": np}) arr = df.values.astype(np.float64, copy=False)
except Exception: vec_expr = expression
# 回退:返回全 NaN 列 for var_name in sorted(var_col_map.keys(), key=len, reverse=True):
result = np.full(len(df), np.nan) col_idx = var_col_map[var_name]
if col_idx < 0 or col_idx >= arr.shape[1]:
raise ValueError(f'列索引 {col_idx} 越界')
vec_expr = re.sub(
r'\b' + re.escape(var_name) + r'\b',
f'arr[:, {col_idx}]',
vec_expr,
)
with np.errstate(divide='ignore', invalid='ignore'):
result = eval(vec_expr, {'__builtins__': None}, {'arr': arr, 'np': np})
result = np.asarray(result, dtype=np.float64).ravel().copy()
if len(result) == len(df) and np.isfinite(result).sum() > 0:
return pd.Series(result, index=df.index, name=expression)
except Exception:
pass # 向量化失败 → 回退逐行
# 确保结果是一维的 # ── 慢车道:逐行 eval永远不会崩──
result = np.asarray(result).ravel() n_rows = len(df)
return pd.Series(result, index=df.index, name=expression) results = [np.nan] * n_rows
for i in range(n_rows):
calc_expr = expression
for var_pattern, col_idx in var_col_map.items():
value = df.iloc[i, col_idx]
calc_expr = re.sub(
r'\b' + re.escape(var_pattern) + r'\b',
f'({value})',
calc_expr,
)
try:
with np.errstate(divide='ignore', invalid='ignore'):
r = eval(calc_expr, {'__builtins__': None}, {'nan': np.nan, 'inf': np.inf, 'np': np})
except Exception:
r = np.nan
results[i] = r
def calculate_one(self, name: str, df: pd.DataFrame, wavelength_offset: float = 0.0) -> pd.Series: return pd.Series(results, index=df.index, name=expression)
def calculate_one(self, name: str, df: pd.DataFrame, wavelength_offset: float = 0.0,
fast: bool = False) -> pd.Series:
""" """
计算单个水质指数。 计算单个水质指数。
@ -163,17 +183,22 @@ class WaterQualityIndexCalculator:
ftype = cfg["type"] ftype = cfg["type"]
coeff_str = cfg["coeff"] coeff_str = cfg["coeff"]
raw = self._band_math_all_rows(df, expr, wavelength_offset=wavelength_offset) raw = self._band_math_all_rows(df, expr, wavelength_offset=wavelength_offset, fast=fast)
if ftype == "concentration": if ftype == "concentration":
coeff = self._parse_coeff(coeff_str) coeff = self._parse_coeff(coeff_str)
result = np.polyval(coeff, raw.values) # ★ .copy() 强制物化polyval 返回的数组可能引用临时内存
result = np.asarray(np.polyval(coeff, raw.values), dtype=np.float64).copy()
raw = pd.Series(result, index=df.index, name=name) raw = pd.Series(result, index=df.index, name=name)
# ★ 强制物化:向量化路径产生的 Series 内部数组可能指向临时内存,
# 后续 csv_processor 的 .values / .replace / .clip 访问会触发 0xC0000005
raw = raw.copy()
raw.name = name raw.name = name
return raw return raw
def calculate_many(self, names: List[str], df: pd.DataFrame, wavelength_offset: float = 0.0) -> pd.DataFrame: def calculate_many(self, names: List[str], df: pd.DataFrame, wavelength_offset: float = 0.0,
fast: bool = False) -> pd.DataFrame:
""" """
批量计算多个水质指数。 批量计算多个水质指数。
@ -181,19 +206,17 @@ class WaterQualityIndexCalculator:
names: 公式名称列表 names: 公式名称列表
df: 光谱反射率 DataFrame df: 光谱反射率 DataFrame
wavelength_offset: 波长偏移修正量nm wavelength_offset: 波长偏移修正量nm
fast: True=优先向量化Step9用False=纯逐行稳定Step10用
Returns:
pd.DataFrame每列对应一个公式的计算结果
""" """
results = {} results = {}
for name in names: for name in names:
try: try:
results[name] = self.calculate_one(name, df, wavelength_offset=wavelength_offset) results[name] = self.calculate_one(name, df, wavelength_offset=wavelength_offset, fast=fast)
except Exception as e: except Exception as e:
print(f"⚠️ 计算 {name} 失败: {e}") print(f"⚠️ 计算 {name} 失败: {e}")
results[name] = pd.Series(np.nan, index=df.index, name=name) results[name] = pd.Series(np.nan, index=df.index, name=name)
return pd.DataFrame(results) return pd.DataFrame(results).copy() if fast else pd.DataFrame(results)
# ========================================================================= # =========================================================================
# 向后兼容:保留原有外部调用接口 # 向后兼容:保留原有外部调用接口