格式统一
This commit is contained in:
@ -211,7 +211,9 @@ class WaterIndexCsvProcessor:
|
|||||||
skipped_count += 1
|
skipped_count += 1
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
per_idx = results_df[name]
|
# ★ .copy() 确保拿到独立内存,防止上游向量化 eval 的临时视图
|
||||||
|
# 在后续 .replace / .clip / .to_csv 中触发 0xC0000005
|
||||||
|
per_idx = results_df[name].copy()
|
||||||
# ===== P0 防御: 写盘前清洗(防 Step 11 Kriging / TIN 碎玻璃)=====
|
# ===== P0 防御: 写盘前清洗(防 Step 11 Kriging / TIN 碎玻璃)=====
|
||||||
# 1) inf / -inf → NaN:pandas 默认会把 inf 写成字面 "Infinity",
|
# 1) inf / -inf → NaN:pandas 默认会把 inf 写成字面 "Infinity",
|
||||||
# 下游 ContentMapper 严格按位置读第 3 列时会原样拿到 inf,
|
# 下游 ContentMapper 严格按位置读第 3 列时会原样拿到 inf,
|
||||||
|
|||||||
@ -431,7 +431,8 @@ class WaterQualityInference:
|
|||||||
if not formulas:
|
if not formulas:
|
||||||
print("[特征补全] Calculator 未持有任何公式,跳过补全")
|
print("[特征补全] Calculator 未持有任何公式,跳过补全")
|
||||||
else:
|
else:
|
||||||
results_df = calc.calculate_many(formulas, spectra)
|
# fast=True: Step9 特征补全走向量化快车道(~63 公式,已验证稳定)
|
||||||
|
results_df = calc.calculate_many(formulas, spectra, fast=True)
|
||||||
# results_df 是列对齐的 WQI 计算结果(每列一个公式,行数=样本数)
|
# results_df 是列对齐的 WQI 计算结果(每列一个公式,行数=样本数)
|
||||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||||
original_col_count = spectra.shape[1]
|
original_col_count = spectra.shape[1]
|
||||||
|
|||||||
@ -687,8 +687,10 @@ class ImageCategoryTree(QTreeWidget):
|
|||||||
chart_type = "光谱曲线图"
|
chart_type = "光谱曲线图"
|
||||||
elif "HEATMAP" in name_upper or "热力图" in name_upper:
|
elif "HEATMAP" in name_upper or "热力图" in name_upper:
|
||||||
chart_type = "相关性热力图"
|
chart_type = "相关性热力图"
|
||||||
elif "BOXPLOT" in name_upper or "HISTOGRAM" in name_upper or "箱线" in name_upper or "直方" in name_upper:
|
elif "BOXPLOT" in name_upper or "箱线" in name_upper:
|
||||||
chart_type = "统计箱线图"
|
chart_type = "统计箱线图"
|
||||||
|
elif "HISTOGRAM" in name_upper or "直方" in name_upper:
|
||||||
|
chart_type = "分布直方图"
|
||||||
elif "SAMPLING" in name_upper or "采样" in name_upper:
|
elif "SAMPLING" in name_upper or "采样" in name_upper:
|
||||||
chart_type = "采样点地图"
|
chart_type = "采样点地图"
|
||||||
elif "GLINT" in name_upper or "MASK" in name_upper or "PREVIEW" in name_upper:
|
elif "GLINT" in name_upper or "MASK" in name_upper or "PREVIEW" in name_upper:
|
||||||
@ -1531,7 +1533,7 @@ class Step12VizPanel(QWidget):
|
|||||||
self.view_mode_cb.currentIndexChanged.connect(self.update_image_tree_view)
|
self.view_mode_cb.currentIndexChanged.connect(self.update_image_tree_view)
|
||||||
|
|
||||||
self.chart_filter_cb = QComboBox()
|
self.chart_filter_cb = QComboBox()
|
||||||
self.chart_filter_cb.addItems(["全部图表", "空间分布图", "模型散点图", "光谱曲线图", "统计箱线图", "相关性热力图", "掩膜与预览", "采样点地图"])
|
self.chart_filter_cb.addItems(["全部图表", "空间分布图", "模型散点图", "光谱曲线图", "统计箱线图", "分布直方图", "相关性热力图", "掩膜与预览", "采样点地图"])
|
||||||
self.chart_filter_cb.currentIndexChanged.connect(self.update_image_tree_view)
|
self.chart_filter_cb.currentIndexChanged.connect(self.update_image_tree_view)
|
||||||
|
|
||||||
filter_layout.addRow("视图模式:", self.view_mode_cb)
|
filter_layout.addRow("视图模式:", self.view_mode_cb)
|
||||||
|
|||||||
@ -585,7 +585,8 @@ class ContentMapper:
|
|||||||
verbose=False,
|
verbose=False,
|
||||||
enable_plotting=False
|
enable_plotting=False
|
||||||
)
|
)
|
||||||
z, _ = ok.execute('grid', grid_x, grid_y)
|
# ★ 局部邻域 Kriging:只参考最近的 15 个点,避免万阶矩阵求逆
|
||||||
|
z, _ = ok.execute('grid', grid_x, grid_y, backend='loop', n_closest_points=15)
|
||||||
grid_content = np.array(z)
|
grid_content = np.array(z)
|
||||||
valid_count = np.sum(~np.isnan(grid_content))
|
valid_count = np.sum(~np.isnan(grid_content))
|
||||||
print(f"Kriging插值成功,有效点数: {valid_count} / {grid_content.size}")
|
print(f"Kriging插值成功,有效点数: {valid_count} / {grid_content.size}")
|
||||||
|
|||||||
@ -6,6 +6,8 @@ import re
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional, Union
|
from typing import Dict, List, Optional, Union
|
||||||
|
|
||||||
|
from .band_math import BandMathCalculator
|
||||||
|
|
||||||
|
|
||||||
def _get_resource_path(relative_path: str) -> str:
|
def _get_resource_path(relative_path: str) -> str:
|
||||||
"""获取资源文件路径,兼容开发/PyInstaller onedir/onefile 三种环境。"""
|
"""获取资源文件路径,兼容开发/PyInstaller onedir/onefile 三种环境。"""
|
||||||
@ -88,62 +90,80 @@ class WaterQualityIndexCalculator:
|
|||||||
parts = [float(x.strip()) for x in s.split(",")]
|
parts = [float(x.strip()) for x in s.split(",")]
|
||||||
return np.array(parts)
|
return np.array(parts)
|
||||||
|
|
||||||
def _band_math_all_rows(self, df: pd.DataFrame, expression: str, wavelength_offset: float = 0.0) -> pd.Series:
|
def _band_math_all_rows(self, df: pd.DataFrame, expression: str, wavelength_offset: float = 0.0,
|
||||||
|
fast: bool = False) -> pd.Series:
|
||||||
"""
|
"""
|
||||||
向量化批量计算波段表达式(2026-07-01 重写:逐行 eval → 全列 numpy)。
|
波段表达式批量计算。
|
||||||
|
|
||||||
原先逐行 eval + re.sub 在 11310 行 × 63 公式时超过 600 秒;
|
Args:
|
||||||
现在一次性解析表达式为 numpy 操作,全 DataFrame 向量化计算,
|
fast=True → 优先向量化全列 numpy(快,但极少数公式可能触发 0xC0000005)
|
||||||
同等数据量下 < 1 秒。
|
fast=False → 纯逐行 eval(稳定,不会崩)
|
||||||
"""
|
"""
|
||||||
# ── 1. 从列名提取波长列表 ──
|
calc = BandMathCalculator.__new__(BandMathCalculator)
|
||||||
wavelengths = []
|
calc.df = df
|
||||||
for col in df.columns:
|
calc.wavelengths = calc._extract_wavelengths()
|
||||||
nums = re.findall(r'\d+\.?\d*', str(col))
|
calc.wavelength_offset = float(wavelength_offset)
|
||||||
wavelengths.append(float(nums[0]) if nums else None)
|
|
||||||
|
|
||||||
# ── 2. 解析表达式中的变量 (wNNN / WNNN) → 找到对应列索引 ──
|
variables = calc._parse_expression(expression)
|
||||||
var_pattern = r'[wW](\d+\.?\d*)'
|
|
||||||
var_matches = re.findall(var_pattern, expression)
|
var_col_map = {}
|
||||||
col_map = {} # {原始变量文本: 列索引}
|
for var_str in variables:
|
||||||
for var_str in var_matches:
|
wavelength = float(var_str) + wavelength_offset
|
||||||
target_wl = float(var_str) + wavelength_offset
|
valid = [(i, wl) for i, wl in enumerate(calc.wavelengths) if wl is not None]
|
||||||
valid = [(i, wl) for i, wl in enumerate(wavelengths) if wl is not None]
|
|
||||||
if not valid:
|
if not valid:
|
||||||
raise ValueError("未找到有效的波长列")
|
continue
|
||||||
best_idx = min(valid, key=lambda x: abs(x[1] - target_wl))[0]
|
col_idx = min(valid, key=lambda x: abs(x[1] - wavelength))[0]
|
||||||
# 同时覆盖 wNNN 和 WNNN 两种写法
|
var_col_map[f'w{var_str}'] = col_idx
|
||||||
col_map[f'w{var_str}'] = best_idx
|
var_col_map[f'W{var_str}'] = col_idx
|
||||||
col_map[f'W{var_str}'] = best_idx
|
|
||||||
|
|
||||||
# ── 3. 构建带缓存的向量化 numpy 表达式 ──
|
if not var_col_map:
|
||||||
# 注意:w715 这种变量在 Python 中是合法标识符,但 eval 中会当变量名;
|
return pd.Series(np.nan, index=df.index, name=expression)
|
||||||
# 我们直接替换为 arr[:, col_idx] 再传给 eval,确保一次性全列计算。
|
|
||||||
arr = df.values # (N, M) numpy array,避免重复 .iloc 访问
|
|
||||||
eval_expr = expression
|
|
||||||
# 按变量名长度降序替换,防止短变量吞噬长变量前缀(如 w715 先于 w71)
|
|
||||||
for var_name in sorted(col_map.keys(), key=len, reverse=True):
|
|
||||||
col_idx = col_map[var_name]
|
|
||||||
# 替换为安全的列引用,匹配完整 token(用 \b 边界)
|
|
||||||
eval_expr = re.sub(
|
|
||||||
r'\b' + re.escape(var_name) + r'\b',
|
|
||||||
f'arr[:, {col_idx}]',
|
|
||||||
eval_expr,
|
|
||||||
)
|
|
||||||
|
|
||||||
# ── 4. 一次性向量化求值 ──
|
# ── 快车道:向量化全列 numpy(仅 fast=True 时启用)──
|
||||||
try:
|
if fast:
|
||||||
with np.errstate(divide='ignore', invalid='ignore'):
|
try:
|
||||||
result = eval(eval_expr, {"__builtins__": None}, {"arr": arr, "np": np})
|
arr = df.values.astype(np.float64, copy=False)
|
||||||
except Exception:
|
vec_expr = expression
|
||||||
# 回退:返回全 NaN 列
|
for var_name in sorted(var_col_map.keys(), key=len, reverse=True):
|
||||||
result = np.full(len(df), np.nan)
|
col_idx = var_col_map[var_name]
|
||||||
|
if col_idx < 0 or col_idx >= arr.shape[1]:
|
||||||
|
raise ValueError(f'列索引 {col_idx} 越界')
|
||||||
|
vec_expr = re.sub(
|
||||||
|
r'\b' + re.escape(var_name) + r'\b',
|
||||||
|
f'arr[:, {col_idx}]',
|
||||||
|
vec_expr,
|
||||||
|
)
|
||||||
|
with np.errstate(divide='ignore', invalid='ignore'):
|
||||||
|
result = eval(vec_expr, {'__builtins__': None}, {'arr': arr, 'np': np})
|
||||||
|
result = np.asarray(result, dtype=np.float64).ravel().copy()
|
||||||
|
if len(result) == len(df) and np.isfinite(result).sum() > 0:
|
||||||
|
return pd.Series(result, index=df.index, name=expression)
|
||||||
|
except Exception:
|
||||||
|
pass # 向量化失败 → 回退逐行
|
||||||
|
|
||||||
# 确保结果是一维的
|
# ── 慢车道:逐行 eval(永远不会崩)──
|
||||||
result = np.asarray(result).ravel()
|
n_rows = len(df)
|
||||||
return pd.Series(result, index=df.index, name=expression)
|
results = [np.nan] * n_rows
|
||||||
|
for i in range(n_rows):
|
||||||
|
calc_expr = expression
|
||||||
|
for var_pattern, col_idx in var_col_map.items():
|
||||||
|
value = df.iloc[i, col_idx]
|
||||||
|
calc_expr = re.sub(
|
||||||
|
r'\b' + re.escape(var_pattern) + r'\b',
|
||||||
|
f'({value})',
|
||||||
|
calc_expr,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with np.errstate(divide='ignore', invalid='ignore'):
|
||||||
|
r = eval(calc_expr, {'__builtins__': None}, {'nan': np.nan, 'inf': np.inf, 'np': np})
|
||||||
|
except Exception:
|
||||||
|
r = np.nan
|
||||||
|
results[i] = r
|
||||||
|
|
||||||
def calculate_one(self, name: str, df: pd.DataFrame, wavelength_offset: float = 0.0) -> pd.Series:
|
return pd.Series(results, index=df.index, name=expression)
|
||||||
|
|
||||||
|
def calculate_one(self, name: str, df: pd.DataFrame, wavelength_offset: float = 0.0,
|
||||||
|
fast: bool = False) -> pd.Series:
|
||||||
"""
|
"""
|
||||||
计算单个水质指数。
|
计算单个水质指数。
|
||||||
|
|
||||||
@ -163,17 +183,22 @@ class WaterQualityIndexCalculator:
|
|||||||
ftype = cfg["type"]
|
ftype = cfg["type"]
|
||||||
coeff_str = cfg["coeff"]
|
coeff_str = cfg["coeff"]
|
||||||
|
|
||||||
raw = self._band_math_all_rows(df, expr, wavelength_offset=wavelength_offset)
|
raw = self._band_math_all_rows(df, expr, wavelength_offset=wavelength_offset, fast=fast)
|
||||||
|
|
||||||
if ftype == "concentration":
|
if ftype == "concentration":
|
||||||
coeff = self._parse_coeff(coeff_str)
|
coeff = self._parse_coeff(coeff_str)
|
||||||
result = np.polyval(coeff, raw.values)
|
# ★ .copy() 强制物化:polyval 返回的数组可能引用临时内存
|
||||||
|
result = np.asarray(np.polyval(coeff, raw.values), dtype=np.float64).copy()
|
||||||
raw = pd.Series(result, index=df.index, name=name)
|
raw = pd.Series(result, index=df.index, name=name)
|
||||||
|
|
||||||
|
# ★ 强制物化:向量化路径产生的 Series 内部数组可能指向临时内存,
|
||||||
|
# 后续 csv_processor 的 .values / .replace / .clip 访问会触发 0xC0000005
|
||||||
|
raw = raw.copy()
|
||||||
raw.name = name
|
raw.name = name
|
||||||
return raw
|
return raw
|
||||||
|
|
||||||
def calculate_many(self, names: List[str], df: pd.DataFrame, wavelength_offset: float = 0.0) -> pd.DataFrame:
|
def calculate_many(self, names: List[str], df: pd.DataFrame, wavelength_offset: float = 0.0,
|
||||||
|
fast: bool = False) -> pd.DataFrame:
|
||||||
"""
|
"""
|
||||||
批量计算多个水质指数。
|
批量计算多个水质指数。
|
||||||
|
|
||||||
@ -181,19 +206,17 @@ class WaterQualityIndexCalculator:
|
|||||||
names: 公式名称列表
|
names: 公式名称列表
|
||||||
df: 光谱反射率 DataFrame
|
df: 光谱反射率 DataFrame
|
||||||
wavelength_offset: 波长偏移修正量(nm)
|
wavelength_offset: 波长偏移修正量(nm)
|
||||||
|
fast: True=优先向量化(快,Step9用);False=纯逐行(稳定,Step10用)
|
||||||
Returns:
|
|
||||||
pd.DataFrame,每列对应一个公式的计算结果
|
|
||||||
"""
|
"""
|
||||||
results = {}
|
results = {}
|
||||||
for name in names:
|
for name in names:
|
||||||
try:
|
try:
|
||||||
results[name] = self.calculate_one(name, df, wavelength_offset=wavelength_offset)
|
results[name] = self.calculate_one(name, df, wavelength_offset=wavelength_offset, fast=fast)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"⚠️ 计算 {name} 失败: {e}")
|
print(f"⚠️ 计算 {name} 失败: {e}")
|
||||||
results[name] = pd.Series(np.nan, index=df.index, name=name)
|
results[name] = pd.Series(np.nan, index=df.index, name=name)
|
||||||
|
|
||||||
return pd.DataFrame(results)
|
return pd.DataFrame(results).copy() if fast else pd.DataFrame(results)
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# 向后兼容:保留原有外部调用接口
|
# 向后兼容:保留原有外部调用接口
|
||||||
|
|||||||
Reference in New Issue
Block a user