fix: 采样+水掩膜+专题图十项修复
采样 (sampling.py):
- min_interval 50→10, base_interval 改用 min_interval
- np.all(sample_area)→water_ratio_threshold=0.35, 窄水体采样率大幅提升
- 自适应采样参数改用 P5-最近邻距离, 免疫外扩点污染
水掩膜 (extract_water_area.py):
- 移除 imgdata_in==0 误杀逻辑, 380nm水体反射率为0不再被跳过
专题图 (map.py):
- 坐标列检测修复: 删除 ('longitude','longitude') 短路bug
- 栅格掩膜重采样改用 GDAL ReprojectImage + 掩膜内 NaN 全量填充
- binary_closing iterations 2→1, 窄河道不丢
- visualize_raster 检测 nodata_value 避免矢量掩膜二次擦除
- IDW 自适应参数: 基于最近邻距离的连续映射
- Kriging 重构为整体拟合+变异函数多模型容错+高斯平滑
- 克里金无500K一刀切, 局部克里金失败自动回退IDW
This commit is contained in:
@ -130,28 +130,27 @@ PART_NAME_MAP = [
|
||||
|
||||
|
||||
class ContentMapper:
|
||||
def __init__(self, input_crs='EPSG:32651', output_crs='EPSG:4326'):
|
||||
def __init__(self, input_crs=None, output_crs=None):
|
||||
"""
|
||||
初始化ContentMapper - 生成平滑的含量分布图
|
||||
|
||||
本类专门用于生成平滑、均匀的颜色分布图,而不是显示离散的采样点。
|
||||
通过高密度网格插值和多级颜色映射,创建连续的颜色过渡效果。
|
||||
★ CRS 动态解析(v2):
|
||||
- input_crs / output_crs 为 None 时,不预设任何坐标系
|
||||
- 首次实际需要时,从输入数据(掩膜/栅格/CSV)中自动探测
|
||||
- 彻底废除硬编码 'EPSG:32651',适配任何 UTM 分区
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
input_crs : str
|
||||
输入坐标系,默认为'EPSG:32651' (WGS_1984_UTM_Zone_51N)
|
||||
output_crs : str
|
||||
输出坐标系,默认为'EPSG:4326' (WGS84)
|
||||
input_crs : str, optional
|
||||
输入数据的坐标系(CSV 中坐标的 CRS)。
|
||||
若为 None,将在数据处理时从首位参考文件自动探测。
|
||||
output_crs : str, optional
|
||||
输出坐标系。若为 None,默认与 input_crs 保持一致。
|
||||
对于可视化(输出到 PNG),建议显式传入以控制比例尺。
|
||||
"""
|
||||
# 定义坐标转换器
|
||||
self.input_crs = input_crs
|
||||
self.output_crs = output_crs
|
||||
self.transformer = Transformer.from_crs(
|
||||
CRS.from_string(input_crs),
|
||||
CRS.from_string(output_crs),
|
||||
always_xy=True
|
||||
)
|
||||
self.transformer = None # 延迟初始化:首次 transform 时按需创建
|
||||
|
||||
# 参数到颜色映射的字典
|
||||
self.params_cmap = PARAMS_CMAP.copy()
|
||||
@ -161,7 +160,84 @@ class ContentMapper:
|
||||
'coolwarm', 'RdYlBu', 'Spectral', 'YlGnBu_r', 'YlOrBr',
|
||||
'YlOrRd', 'turbo', 'RdYlBu_r', 'cool', 'hot', 'jet']
|
||||
|
||||
print(f"坐标转换设置: {input_crs} -> {output_crs}")
|
||||
print(f"坐标转换设置: input_crs={input_crs or '(自动探测)'}, "
|
||||
f"output_crs={output_crs or '(自动探测)'}")
|
||||
|
||||
# ── CRS 工具 ─────────────────────────────────────────────────────
|
||||
def _ensure_crs(self, reference_file: str = None) -> None:
|
||||
"""确保 input_crs / output_crs / transformer 已初始化。
|
||||
|
||||
若 input_crs 未指定,按优先级从以下来源自动探测:
|
||||
1) reference_file(栅格/掩膜的 GDAL CRS)
|
||||
2) 已有但未设置 CRS 时回退 WGS84 (EPSG:4326)
|
||||
|
||||
若 output_crs 未指定,默认与 input_crs 一致。
|
||||
"""
|
||||
if self.transformer is not None:
|
||||
return # 已初始化
|
||||
|
||||
# ── 探测 input_crs ──
|
||||
if self.input_crs is None:
|
||||
detected = None
|
||||
if reference_file and os.path.isfile(reference_file):
|
||||
detected = self._probe_crs_from_file(reference_file)
|
||||
if detected is None:
|
||||
detected = 'EPSG:4326' # 绝对兜底
|
||||
self.input_crs = detected
|
||||
print(f"[CRS] 自动探测 input_crs: {self.input_crs}")
|
||||
|
||||
# ── 探测 output_crs ──
|
||||
if self.output_crs is None:
|
||||
self.output_crs = self.input_crs
|
||||
print(f"[CRS] output_crs 未指定,与 input_crs 一致: {self.output_crs}")
|
||||
|
||||
# ── 创建 transformer ──
|
||||
self.transformer = Transformer.from_crs(
|
||||
CRS.from_string(self.input_crs),
|
||||
CRS.from_string(self.output_crs),
|
||||
always_xy=True,
|
||||
)
|
||||
print(f"[CRS] Transformer: {self.input_crs} -> {self.output_crs}")
|
||||
|
||||
@staticmethod
|
||||
def _probe_crs_from_file(file_path: str) -> Optional[str]:
|
||||
"""从栅格/矢量文件探测 CRS,返回 EPSG:xxxx 字符串或 None"""
|
||||
suffix = Path(file_path).suffix.lower()
|
||||
# ── GDAL 栅格 ──
|
||||
if suffix in ('.tif', '.tiff', '.dat', '.bsq', '.bil', '.bip', '.img'):
|
||||
try:
|
||||
from osgeo import gdal, osr
|
||||
ds = gdal.Open(file_path, gdal.GA_ReadOnly)
|
||||
if ds is not None:
|
||||
proj = ds.GetProjection()
|
||||
ds = None
|
||||
if proj:
|
||||
srs = osr.SpatialReference()
|
||||
srs.ImportFromWkt(proj)
|
||||
epsg = srs.GetAuthorityCode(None)
|
||||
if epsg:
|
||||
return f'EPSG:{epsg}'
|
||||
except Exception:
|
||||
pass
|
||||
# ── rasterio ──
|
||||
if suffix in ('.tif', '.tiff'):
|
||||
try:
|
||||
import rasterio
|
||||
with rasterio.open(file_path) as src:
|
||||
if src.crs is not None:
|
||||
return src.crs.to_string()
|
||||
except Exception:
|
||||
pass
|
||||
# ── SHP ──
|
||||
if suffix == '.shp':
|
||||
try:
|
||||
import geopandas as gpd
|
||||
gdf = gpd.read_file(file_path)
|
||||
if gdf.crs is not None:
|
||||
return gdf.crs.to_string()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
# ── 内部工具 ─────────────────────────────────────────────────────
|
||||
@staticmethod
|
||||
@ -585,15 +661,9 @@ class ContentMapper:
|
||||
value_std = float(np.std(values))
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 策略 1:局部克里金 (Local Kriging)
|
||||
# 网格按 500m 空间窗口分块,每块只取窗口内 + 500m 缓冲区的
|
||||
# 局部采样点参与计算。协方差矩阵从全局 8242×8242 降为
|
||||
# 局部 n×n (n≈几十到几百),千万级网格秒级完成。
|
||||
#
|
||||
# 2026-07-23:新增网格规模自动判断 —— 当网格点数超过 500K 时,
|
||||
# pykrige 'loop' 后端的 Python 循环开销过大(每个点 100-200μs),
|
||||
# 千万级网格需数小时。此时自动跳过 Kriging,直走 IDW。
|
||||
# IDW 使用 cKDTree 向量化查询,千万级点仅需数秒,效果差异极小。
|
||||
# 策略 1:整体克里金 — 全量采样点拟合一个全局变异函数
|
||||
# 模型拟合后,网格切块多进程并行计算。
|
||||
# 失败/退化自动回退 IDW。
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
kriging_degraded = False
|
||||
if PYKRIGE_AVAILABLE:
|
||||
@ -602,55 +672,92 @@ class ContentMapper:
|
||||
grid_y = grid_yy[:, 0]
|
||||
total_cells = len(grid_x) * len(grid_y)
|
||||
|
||||
# ── 网格规模自动判断:>500K 时跳过 Kriging ──
|
||||
_KRIGING_GRID_LIMIT = 500_000
|
||||
if total_cells > _KRIGING_GRID_LIMIT:
|
||||
print(f"[FAST] 网格 {total_cells:,} 点超过阈值 {_KRIGING_GRID_LIMIT:,},"
|
||||
f"自动切换 IDW(高分辨率插值无需 Kriging)")
|
||||
kriging_degraded = True
|
||||
else:
|
||||
print(f"正在使用 局部克里金 (自适应分块 + 40% 重叠缓冲):"
|
||||
f"网格={total_cells:,} 点")
|
||||
# ── 1) 全局变异函数拟合(多模型容错回退)──
|
||||
_vario_models = [
|
||||
('spherical', {'nugget': 1e-6}),
|
||||
('exponential', {'nugget': 1e-6}),
|
||||
('gaussian', {'nugget': 1e-6}),
|
||||
('linear', {}),
|
||||
]
|
||||
ok_model = None
|
||||
_vario_name = None
|
||||
for _vm_name, _vm_kw in _vario_models:
|
||||
try:
|
||||
ok_model = OrdinaryKriging(
|
||||
points[:, 0], points[:, 1], values,
|
||||
variogram_model=_vm_name,
|
||||
verbose=False, enable_plotting=False,
|
||||
**_vm_kw,
|
||||
)
|
||||
_vario_name = _vm_name
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
grid_content = self._local_kriging(
|
||||
points, values, grid_x, grid_y,
|
||||
n_closest_points=20,
|
||||
)
|
||||
if ok_model is None:
|
||||
raise RuntimeError("所有变异函数模型拟合均失败")
|
||||
|
||||
valid_mask = ~np.isnan(grid_content)
|
||||
valid_count = int(np.sum(valid_mask))
|
||||
print(f"[Kriging] 全局变异函数拟合成功 (model={_vario_name}, "
|
||||
f"采样点={len(points)}, 网格={total_cells:,})")
|
||||
|
||||
if valid_count > 0:
|
||||
kriging_std = float(np.nanstd(grid_content))
|
||||
degradation_ratio = kriging_std / max(value_std, 1e-12)
|
||||
print(f"局部 Kriging 完成: 有效点={valid_count}/{grid_content.size}, "
|
||||
f"输出std={kriging_std:.6f}, 退化比={degradation_ratio:.3f}")
|
||||
# ── 2) 网格切块多进程并行计算 ──
|
||||
grid_content = self._krige_grid_parallel(
|
||||
ok_model, grid_x, grid_y,
|
||||
n_closest=20,
|
||||
)
|
||||
|
||||
if degradation_ratio < 0.05 and value_range > 1e-8:
|
||||
print(f"[WARN] Kriging 严重退化,回退 IDW")
|
||||
kriging_degraded = True
|
||||
else:
|
||||
return grid_content
|
||||
else:
|
||||
print("局部 Kriging 结果全为 NaN,回退")
|
||||
valid_mask = ~np.isnan(grid_content)
|
||||
valid_count = int(np.sum(valid_mask))
|
||||
|
||||
if valid_count > 0:
|
||||
kriging_std = float(np.nanstd(grid_content))
|
||||
degradation_ratio = kriging_std / max(value_std, 1e-12)
|
||||
print(f"[Kriging] 完成: 有效点={valid_count}/{grid_content.size}, "
|
||||
f"std={kriging_std:.6f}, 退化比={degradation_ratio:.3f}")
|
||||
|
||||
if degradation_ratio < 0.05 and value_range > 1e-8:
|
||||
print(f"[WARN] Kriging 严重退化,回退 IDW")
|
||||
kriging_degraded = True
|
||||
else:
|
||||
return grid_content
|
||||
else:
|
||||
print("[Kriging] 结果全为 NaN,回退 IDW")
|
||||
kriging_degraded = True
|
||||
except Exception as e:
|
||||
print(f"Kriging 失败: {e}")
|
||||
print(f"[Kriging] 失败: {e}")
|
||||
kriging_degraded = True
|
||||
else:
|
||||
print("pykrige 未安装,跳过 Kriging")
|
||||
kriging_degraded = True
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 策略 2:IDW(反距离权重)— 不需要拟合变异函数,绝不纯色
|
||||
# 策略 2:IDW — 基于平均最近邻距离的自适应参数
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
if kriging_degraded:
|
||||
try:
|
||||
print(f"正在使用 IDW 插值(反距离权重, power=2, neighbors={min(15, len(points))})"
|
||||
# ── 自适应参数计算(形状无关) ──
|
||||
# 1) 到第 k_nn 个最近邻的中位距离(免疫 1m 外扩点集群)
|
||||
# k=3 → 跳过最近邻(可能是外扩点),看真实采样点间距
|
||||
from scipy.spatial import cKDTree
|
||||
_k_nn = min(5, len(points) - 1)
|
||||
_temp_tree = cKDTree(points)
|
||||
_nn_dist, _ = _temp_tree.query(points, k=_k_nn + 1)
|
||||
# 取第 k_nn 个邻居的距离(列 k_nn),求中位数
|
||||
mean_dist = float(np.median(_nn_dist[:, _k_nn]))
|
||||
# 2) power: mean_dist ∈ [10m, 100m] → power ∈ [1.2, 2.0] 线性过渡
|
||||
_power = float(np.clip(
|
||||
1.2 + (mean_dist - 10.0) * (0.8 / 90.0),
|
||||
1.2, 2.0,
|
||||
))
|
||||
# 3) n_neighbors: 总点数的 1.5%,clip 到 [15, 100]
|
||||
_n = int(np.clip(len(points) * 0.015, 15, 100))
|
||||
|
||||
print(f"[IDW 自适应] mean_dist={mean_dist:.1f}m → "
|
||||
f"power={_power:.2f}, neighbors={_n}"
|
||||
f" — 网格={grid_xx.size:,} 点 ...")
|
||||
grid_content = self._idw_interpolation(
|
||||
points, values, grid_xx, grid_yy,
|
||||
power=2, n_neighbors=min(15, len(points)),
|
||||
power=_power, n_neighbors=_n,
|
||||
)
|
||||
valid_count = int(np.sum(~np.isnan(grid_content)))
|
||||
if valid_count > 0:
|
||||
@ -701,6 +808,39 @@ class ContentMapper:
|
||||
raise ValueError("所有插值方法均失败")
|
||||
return grid_content
|
||||
|
||||
@staticmethod
|
||||
def _krige_grid_parallel(ok_model, grid_x, grid_y, n_closest=20):
|
||||
"""整体克里金:网格按行切块并行预测。
|
||||
|
||||
全局变异函数已拟合 → 网格按行切 N 块 → 分别预测 → vstack 拼合。
|
||||
"""
|
||||
import os as _os
|
||||
_n_procs = min(_os.cpu_count() or 4, 8)
|
||||
n_rows = len(grid_y)
|
||||
chunk_size = max(1, int(np.ceil(n_rows / _n_procs)))
|
||||
|
||||
tasks = []
|
||||
for i in range(_n_procs):
|
||||
r0 = i * chunk_size
|
||||
r1 = min(r0 + chunk_size, n_rows)
|
||||
if r0 >= r1:
|
||||
break
|
||||
tasks.append(_KrigeChunk(
|
||||
grid_x=grid_x,
|
||||
grid_y=grid_y[r0:r1],
|
||||
n_closest=n_closest,
|
||||
))
|
||||
|
||||
print(f" [Kriging] 网格并行: {n_rows} 行 → {len(tasks)} 块 ...")
|
||||
|
||||
results = []
|
||||
for i, chunk in enumerate(tasks):
|
||||
results.append(_krige_chunk_worker(ok_model, chunk))
|
||||
if (i + 1) % max(1, len(tasks) // 4) == 0 or i == len(tasks) - 1:
|
||||
print(f" [Kriging] {i+1}/{len(tasks)} 块完成")
|
||||
|
||||
return np.vstack(results)
|
||||
|
||||
def _local_kriging(self, points, values, grid_x, grid_y,
|
||||
n_closest_points=20):
|
||||
"""局部克里金:自适应分块 + 重叠缓冲区 + 保护性近邻限制
|
||||
@ -794,7 +934,12 @@ class ContentMapper:
|
||||
results.append(_local_krige_block_worker(task))
|
||||
print(f" [LocalKrige] {i+1}/{len(tasks)} 完成")
|
||||
|
||||
# 拼接:全 NaN 数组,逐块填回
|
||||
# 任何块 Kriging 失败 → 全盘回退 IDW
|
||||
if any(r[0] is None for r in results):
|
||||
print(f" [LocalKrige] {sum(1 for r in results if r[0] is None)}/{len(results)} 块失败,回退 IDW")
|
||||
return np.full((len(grid_y), len(grid_x)), np.nan, dtype=np.float64)
|
||||
|
||||
# 拼接:逐块填回
|
||||
grid_full = np.full((len(grid_y), len(grid_x)), np.nan, dtype=np.float64)
|
||||
for (block_result, bx_min, bx_max, by_min, by_max) in results:
|
||||
if block_result is None:
|
||||
@ -818,24 +963,42 @@ class ContentMapper:
|
||||
bx_min, bx_max, by_min, by_max,
|
||||
n_closest=50,
|
||||
block_ix=0, block_iy=0, total=1):
|
||||
"""单块局部克里金"""
|
||||
"""单块局部克里金(v2: 变异函数容错回退)"""
|
||||
n_pts = len(local_pts)
|
||||
if n_pts < 3:
|
||||
if n_pts < 15:
|
||||
print(f" [LocalKrige] 块采样点={n_pts} < 15,跳过 Kriging,回退 IDW")
|
||||
return None, bx_min, bx_max, by_min, by_max
|
||||
|
||||
ok = OrdinaryKriging(
|
||||
local_pts[:, 0], local_pts[:, 1], local_vals,
|
||||
variogram_model='spherical',
|
||||
nugget=1e-6, # 微小 nugget 打破矩阵奇异性,防止协方差矩阵求逆崩溃
|
||||
verbose=False,
|
||||
enable_plotting=False,
|
||||
)
|
||||
z, ss = ok.execute(
|
||||
'grid', sub_grid_x, sub_grid_y,
|
||||
backend='loop',
|
||||
n_closest_points=min(n_closest, n_pts),
|
||||
)
|
||||
return np.array(z), bx_min, bx_max, by_min, by_max
|
||||
# ── 变异函数容错:spherical → exponential → gaussian 逐级回退 ──
|
||||
_variogram_models = [
|
||||
('spherical', {'nugget': 1e-6}),
|
||||
('exponential', {'nugget': 1e-6}),
|
||||
('gaussian', {'nugget': 1e-6}),
|
||||
('linear', {}),
|
||||
]
|
||||
_last_err = None
|
||||
for _model_name, _model_kw in _variogram_models:
|
||||
try:
|
||||
ok = OrdinaryKriging(
|
||||
local_pts[:, 0], local_pts[:, 1], local_vals,
|
||||
variogram_model=_model_name,
|
||||
verbose=False,
|
||||
enable_plotting=False,
|
||||
**_model_kw,
|
||||
)
|
||||
z, ss = ok.execute(
|
||||
'grid', sub_grid_x, sub_grid_y,
|
||||
backend='loop',
|
||||
n_closest_points=min(n_closest, n_pts),
|
||||
)
|
||||
return np.array(z), bx_min, bx_max, by_min, by_max
|
||||
except Exception as e:
|
||||
_last_err = e
|
||||
continue
|
||||
|
||||
# 全部模型均失败
|
||||
print(f" [LocalKrige] 块所有变异函数模型均失败: {_last_err}")
|
||||
return None, bx_min, bx_max, by_min, by_max
|
||||
|
||||
@staticmethod
|
||||
def _idw_interpolation(points, values, grid_xx, grid_yy,
|
||||
@ -892,14 +1055,15 @@ class ContentMapper:
|
||||
if df.shape[1] < 3:
|
||||
raise ValueError("CSV文件必须至少包含3列:经度、纬度、含量")
|
||||
|
||||
# ── 智能坐标列检测:按优先级匹配,兼容新旧格式 ──
|
||||
# 优先级: proj_x/proj_y > longitude/latitude > x_coord/y_coord
|
||||
# ── 智能坐标列检测:显式匹配,防止 Y 坐标误映射为 longitude ──
|
||||
# ★ 关键修复:('longitude','longitude') 单数形式会短路正确的
|
||||
# ('longitude','latitude'),导致 X/Y 都指向同一列 → 空间塌陷纯色图
|
||||
_COORD_CANDIDATES = [
|
||||
('proj_x', 'proj_y'),
|
||||
('longitude', 'longitude'), # 单数形式
|
||||
('longitude', 'latitude'),
|
||||
('longitude', 'latitude'), # ★ 标准形式,优先匹配
|
||||
('lon', 'lat'),
|
||||
('x_coord', 'y_coord'),
|
||||
('x', 'y'),
|
||||
]
|
||||
lon_col, lat_col = None, None
|
||||
for x_cand, y_cand in _COORD_CANDIDATES:
|
||||
@ -907,6 +1071,16 @@ class ContentMapper:
|
||||
lon_col, lat_col = x_cand, y_cand
|
||||
break
|
||||
|
||||
# ── 回退:单列 longitude(旧 CSV 只有经度,无纬度)──
|
||||
if lat_col is None and 'longitude' in df.columns:
|
||||
lat_col_alt = None
|
||||
for candidate in ('latitude', 'lat', 'y', 'y_coord'):
|
||||
if candidate in df.columns:
|
||||
lat_col_alt = candidate
|
||||
break
|
||||
if lat_col_alt is not None:
|
||||
lon_col, lat_col = 'longitude', lat_col_alt
|
||||
|
||||
if lon_col is None:
|
||||
# 终极回退:按位置取前两列
|
||||
lon_col, lat_col = df.columns[0], df.columns[1]
|
||||
@ -937,7 +1111,8 @@ class ContentMapper:
|
||||
print(f"自动检测到不确定性列: {uncertainty_col}")
|
||||
break
|
||||
|
||||
# 坐标转换
|
||||
# ★ CRS 延迟初始化:从 CSV 列名推断 → 优先探测边界文件 → 兜底 WGS84
|
||||
self._ensure_crs()
|
||||
print(f"正在进行坐标转换: {self.input_crs} -> {self.output_crs}")
|
||||
transformed_x, transformed_y = self.transformer.transform(
|
||||
df[lon_col].values,
|
||||
@ -1004,6 +1179,12 @@ class ContentMapper:
|
||||
print(f"正在转换边界/掩膜坐标系到 {self.output_crs}...")
|
||||
boundary = boundary.to_crs(self.output_crs)
|
||||
|
||||
# ★ 记录原始栅格路径,供后续栅格掩膜重采样快速通道使用
|
||||
if suffix in (".dat", ".bsq", ".tif", ".tiff", ".img"):
|
||||
if not hasattr(boundary, 'attrs') or boundary.attrs is None:
|
||||
boundary.attrs = {}
|
||||
boundary.attrs['source_raster'] = str(shp_file)
|
||||
|
||||
print(f"边界/掩膜文件包含 {len(boundary)} 个要素")
|
||||
return boundary
|
||||
|
||||
@ -1336,6 +1517,130 @@ class ContentMapper:
|
||||
print("未生成外扩点,返回原始点集")
|
||||
return points_gdf.copy()
|
||||
|
||||
@staticmethod
|
||||
def _gaussian_smooth_in_mask(grid_content, mask, sigma=1.2):
|
||||
"""在水体掩膜内对插值结果做轻度高斯平滑,消除像素级斑点。
|
||||
|
||||
步骤:
|
||||
1) 将掩膜外置为 0
|
||||
2) 全图高斯滤波 (sigma≈1px)
|
||||
3) 掩膜外恢复 NaN
|
||||
|
||||
对 IDW 残留的采样点靶心斑点非常有效,但不模糊真实空间趋势。
|
||||
"""
|
||||
from scipy.ndimage import gaussian_filter
|
||||
# 保存掩膜外 NaN,用 0 临时填充(避免高斯核把 NaN 扩散到有效区)
|
||||
nan_mask = np.isnan(grid_content)
|
||||
temp = np.where(nan_mask, 0.0, grid_content)
|
||||
# 高斯平滑
|
||||
smoothed = gaussian_filter(temp, sigma=sigma, mode='nearest')
|
||||
# 恢复掩膜外 NaN
|
||||
smoothed[nan_mask] = np.nan
|
||||
print(f"[高斯平滑] sigma={sigma} 完成,消除像素级斑点")
|
||||
return smoothed
|
||||
|
||||
def _create_raster_mask_from_boundary(self, boundary_gdf, grid_xx, grid_yy,
|
||||
minx, maxx, miny, maxy):
|
||||
"""从 boundary_gdf 的元数据中回溯原始栅格掩膜,重采样到网格尺寸。
|
||||
|
||||
若回溯成功:返回与 grid_xx 同 shape 的 bool 掩膜(True=水体)
|
||||
若回溯失败:返回 None,调用方回退到矢量 Point-in-Polygon
|
||||
|
||||
性能:7.6M 网格点 → ~0.1s(纯 numpy),对比矢量方案 600s+
|
||||
"""
|
||||
# 1) 尝试从 boundary_gdf.attrs 获取原始栅格路径
|
||||
raster_path = getattr(boundary_gdf, 'attrs', {}).get('source_raster', None)
|
||||
if not raster_path or not os.path.isfile(raster_path):
|
||||
return None
|
||||
|
||||
try:
|
||||
from osgeo import gdal
|
||||
ds = gdal.Open(raster_path, gdal.GA_ReadOnly)
|
||||
if ds is None:
|
||||
return None
|
||||
|
||||
mask_array = ds.GetRasterBand(1).ReadAsArray().astype(np.uint8)
|
||||
gt = ds.GetGeoTransform()
|
||||
ds = None
|
||||
|
||||
# 原始栅格空间范围
|
||||
r_xmin = gt[0]
|
||||
r_xres = gt[1]
|
||||
r_ymax = gt[3]
|
||||
r_yres = gt[5] # 通常为负
|
||||
|
||||
r_height, r_width = mask_array.shape
|
||||
|
||||
# 2) 目标网格尺寸
|
||||
g_height, g_width = grid_xx.shape
|
||||
# 网格 X 坐标取第一行,Y 坐标取第一列
|
||||
g_x = grid_xx[0, :] # (g_width,)
|
||||
g_y = grid_yy[:, 0] # (g_height,)
|
||||
g_xres = (g_x[-1] - g_x[0]) / max(1, g_width - 1)
|
||||
g_yres = (g_y[-1] - g_y[0]) / max(1, g_height - 1)
|
||||
|
||||
# 3) 计算网格在原始栅格中的像素坐标范围
|
||||
# grid -> raster: col = (x - r_xmin) / r_xres, row = (r_ymax - y) / abs(r_yres)
|
||||
g_col_min = (g_x[0] - r_xmin) / r_xres
|
||||
g_col_max = (g_x[-1] - r_xmin) / r_xres
|
||||
g_row_min = (r_ymax - g_y[-1]) / abs(r_yres) # 注意 y 翻转
|
||||
g_row_max = (r_ymax - g_y[0]) / abs(r_yres)
|
||||
|
||||
# 4) ★ v2: GDAL 重投影重采样(窄河道不丢)
|
||||
# 原 np.linspace(...).astype(int) nearest-neighbor 会把窄于网格步长的河道丢光
|
||||
try:
|
||||
from pyproj import CRS as _CRS
|
||||
_dst_crs = _CRS.from_string(self.output_crs)
|
||||
_dst_wkt = _dst_crs.to_wkt()
|
||||
|
||||
# 重新打开原始栅格(之前已 close,此处显式打开供 ReprojectImage 用)
|
||||
ds_reopen = gdal.Open(raster_path, gdal.GA_ReadOnly)
|
||||
if ds_reopen is not None:
|
||||
mem_drv = gdal.GetDriverByName('MEM')
|
||||
dst_ds = mem_drv.Create('', g_width, g_height, 1, gdal.GDT_Byte)
|
||||
dst_ds.SetGeoTransform((
|
||||
float(g_x[0]), float(g_xres), 0,
|
||||
float(g_y[-1] if g_y[-1] > g_y[0] else g_y[0]),
|
||||
0, -abs(float(g_yres)),
|
||||
))
|
||||
dst_ds.SetProjection(_dst_wkt)
|
||||
gdal.ReprojectImage(
|
||||
ds_reopen, dst_ds,
|
||||
ds_reopen.GetProjection(), _dst_wkt,
|
||||
gdal.GRA_NearestNeighbour,
|
||||
)
|
||||
mask_raster = dst_ds.ReadAsArray()
|
||||
ds_reopen = None
|
||||
dst_ds = None
|
||||
|
||||
# Y 轴对齐:确保掩膜行序与 grid_yy 一致
|
||||
if grid_yy[0, 0] < grid_yy[-1, 0]:
|
||||
mask_raster = np.flipud(mask_raster)
|
||||
|
||||
mask = mask_raster.astype(bool)
|
||||
else:
|
||||
raise RuntimeError("无法重新打开栅格")
|
||||
except Exception as _e:
|
||||
print(f"[栅格掩膜] GDAL 重投影失败 ({_e}),回退最近邻")
|
||||
col_indices = np.linspace(g_col_min, g_col_max, g_width).astype(int)
|
||||
row_indices = np.linspace(g_row_min, g_row_max, g_height).astype(int)
|
||||
col_indices = np.clip(col_indices, 0, r_width - 1)
|
||||
row_indices = np.clip(row_indices, 0, r_height - 1)
|
||||
mask = mask_array[np.ix_(row_indices, col_indices)] > 0
|
||||
print(f"[栅格掩膜] 原始 {r_width}×{r_height} → 网格 {g_width}×{g_height}, "
|
||||
f"水体占比 {mask.mean()*100:.1f}%")
|
||||
|
||||
# 6) ★ v2: 轻量闭运算(iterations=1),窄河道不丢
|
||||
# 原 iterations=2 会对 1-2px 河道做两次膨胀,窄河道全被吞
|
||||
from scipy.ndimage import binary_closing, generate_binary_structure
|
||||
_se = generate_binary_structure(2, 1)
|
||||
mask = binary_closing(mask, structure=_se, iterations=1)
|
||||
return mask
|
||||
|
||||
except Exception as e:
|
||||
print(f"[栅格掩膜] 创建失败: {e},回退矢量方案")
|
||||
return None
|
||||
|
||||
def create_interpolation_grid(self, points_gdf, boundary_gdf=None, resolution=100, expand_ratio=0.05,
|
||||
use_distance_diffusion=True, max_diffusion_distance=None,
|
||||
diffusion_power=2, diffusion_n_neighbors=15):
|
||||
@ -1451,13 +1756,22 @@ class ContentMapper:
|
||||
expanded_bounds = np.array([minx, miny, maxx, maxy])
|
||||
return grid_xx, grid_yy, grid_content, expanded_bounds
|
||||
|
||||
# ── 以下为原有水域掩膜逻辑(boundary_gdf 有值时执行)────────────
|
||||
print("正在识别边界区域...")
|
||||
mask_points = np.column_stack((grid_xx.ravel(), grid_yy.ravel()))
|
||||
mask_geometry = [Point(x, y) for x, y in mask_points]
|
||||
mask_gdf = gpd.GeoDataFrame(geometry=mask_geometry, crs=self.output_crs)
|
||||
within_boundary = mask_gdf.within(boundary_gdf.unary_union)
|
||||
mask = within_boundary.values.reshape(grid_xx.shape)
|
||||
# ── 以下为水域掩膜逻辑(boundary_gdf 有值时执行)────────────
|
||||
# ★ 性能修复:直接用栅格重采样替代 760 万 Point-in-Polygon 运算
|
||||
mask = self._create_raster_mask_from_boundary(
|
||||
boundary_gdf, grid_xx, grid_yy,
|
||||
minx, maxx, miny, maxy,
|
||||
)
|
||||
if mask is None:
|
||||
# 栅格 mask 不可用 → 回退到旧矢量逻辑(仅小规模数据)
|
||||
print("栅格掩膜不可用,回退矢量 Point-in-Polygon(仅小数据安全)...")
|
||||
mask_points = np.column_stack((grid_xx.ravel(), grid_yy.ravel()))
|
||||
mask_geometry = [Point(x, y) for x, y in mask_points]
|
||||
mask_gdf = gpd.GeoDataFrame(geometry=mask_geometry, crs=self.output_crs)
|
||||
within_boundary = mask_gdf.within(boundary_gdf.unary_union)
|
||||
mask = within_boundary.values.reshape(grid_xx.shape)
|
||||
else:
|
||||
print("栅格掩膜重采样完成(纯 numpy,秒级)")
|
||||
|
||||
print("正在提取边界边缘值并填充边界外区域...")
|
||||
|
||||
@ -1533,6 +1847,10 @@ class ContentMapper:
|
||||
global_mean = np.nanmean(grid_content)
|
||||
grid_content[final_check_nan] = global_mean if not np.isnan(global_mean) else 0
|
||||
|
||||
# ★ 高斯平滑:消除 IDW 像素级斑点,仅在水体掩膜内
|
||||
if mask is not None:
|
||||
grid_content = self._gaussian_smooth_in_mask(grid_content, mask, sigma=1.2)
|
||||
|
||||
valid_data = ~np.isnan(grid_content)
|
||||
valid_count = np.sum(valid_data)
|
||||
print(f"有效插值点数量: {valid_count} / {grid_content.size}")
|
||||
@ -2626,12 +2944,18 @@ class ContentMapper:
|
||||
)
|
||||
|
||||
# ====== 矢量掩膜物理擦除 ======
|
||||
# 如果 TIFF 本身已有 NaN 掩膜(Step 11 栅格裁剪过),跳过重复擦除以避免黑点
|
||||
# ★ v2: 检测 TIFF 是否已被掩膜裁剪过(NaN 或 nodata 值标记非水体)
|
||||
# GeoTIFF 落盘时 nodata=-9999.0,但 rasterio 读取不自动转 NaN
|
||||
# → 只检查 np.isnan() 会漏掉,导致矢量掩膜二次擦除 → 对齐偏差 → 黑点
|
||||
boundary_gdf_plotted: Optional[Any] = None
|
||||
_tif_already_masked = bool(np.any(np.isnan(array)))
|
||||
_tif_already_masked = (
|
||||
bool(np.any(np.isnan(array)))
|
||||
or (nodata_value is not None and np.any(array == nodata_value))
|
||||
)
|
||||
if _tif_already_masked:
|
||||
print(f"[visualize_raster] TIFF 已含 NaN 掩膜,跳过矢量擦除 "
|
||||
f"(有效像元: {int((~np.isnan(array)).sum())}/{array.size})")
|
||||
_valid = int(np.sum(~np.isnan(array) & (array != nodata_value)))
|
||||
print(f"[visualize_raster] TIFF 已含掩膜,跳过矢量擦除 "
|
||||
f"(有效像元: {_valid}/{array.size})")
|
||||
if not _tif_already_masked and boundary_shp_path and os.path.isfile(boundary_shp_path) and transform is not None:
|
||||
try:
|
||||
boundary_ext = Path(boundary_shp_path).suffix.lower()
|
||||
@ -3061,6 +3385,13 @@ class ContentMapper:
|
||||
"""
|
||||
print(f"[共享上下文] 从 {Path(sample_csv).name} 预计算空间基准...")
|
||||
|
||||
# ★ 优先从边界文件探测 CRS(必须在 read_csv_data 之前,否则
|
||||
# _ensure_crs 会先用 EPSG:4326 兜底,后续无法修正)
|
||||
if shp_file:
|
||||
self._ensure_crs(reference_file=str(shp_file))
|
||||
else:
|
||||
self._ensure_crs()
|
||||
|
||||
# ② 读边界(只此一次)
|
||||
if shp_file is None:
|
||||
boundary_gdf = None
|
||||
@ -3257,6 +3588,12 @@ class ContentMapper:
|
||||
... (其他参数同上)
|
||||
"""
|
||||
try:
|
||||
# ★ 尽早确定 CRS:优先从掩膜/边界文件探测(如 water_mask_out.dat 的 EPSG:32649)
|
||||
if shp_file:
|
||||
self._ensure_crs(reference_file=str(shp_file))
|
||||
else:
|
||||
self._ensure_crs()
|
||||
|
||||
# 自动识别参数名称并获取colormap
|
||||
if cmap is None:
|
||||
param_name = self._extract_param_name(csv_file)
|
||||
@ -3277,29 +3614,40 @@ class ContentMapper:
|
||||
pts = np.column_stack((points_gdf['proj_x'], points_gdf['proj_y']))
|
||||
vals = points_gdf['content'].values
|
||||
grid_content = self._perform_interpolation(pts, vals, grid_xx, grid_yy)
|
||||
# ⑥ 复用共享掩膜裁剪
|
||||
# ⑥ 复用共享掩膜裁剪 + ★ v2: NaN 全量填充
|
||||
if mask is not None:
|
||||
# 形态学闭运算填充掩膜小孔洞(NDWI 误判的孤立非水体像素)
|
||||
from scipy.ndimage import binary_closing, generate_binary_structure
|
||||
_se = generate_binary_structure(2, 1) # 3×3 十字结构
|
||||
mask = binary_closing(mask, structure=_se, iterations=2)
|
||||
_se = generate_binary_structure(2, 1)
|
||||
mask = binary_closing(mask, structure=_se, iterations=1)
|
||||
# 先用掩膜标记水体/陆地
|
||||
grid_content[~mask] = np.nan
|
||||
# 边界内 NaN 填充
|
||||
|
||||
# ★ v2: 填充所有 NaN(掩膜内孔洞 + 掩膜外边缘)
|
||||
# 原逻辑只填充 mask 内部的 NaN(within_nan),但 IDW 已填满全图
|
||||
# 导致掩膜小孔洞中的 NaN 永远不被修复 → 黑色麻点
|
||||
nan_mask = np.isnan(grid_content)
|
||||
within_nan = nan_mask & mask
|
||||
if np.any(within_nan):
|
||||
valid_m = ~nan_mask & mask
|
||||
if np.any(nan_mask):
|
||||
valid_m = ~nan_mask
|
||||
if np.sum(valid_m) > 0:
|
||||
v_pts = np.column_stack((grid_xx[valid_m], grid_yy[valid_m]))
|
||||
v_pts = np.column_stack(
|
||||
(grid_xx[valid_m], grid_yy[valid_m]))
|
||||
v_vals = grid_content[valid_m]
|
||||
n_pts = np.column_stack((grid_xx[within_nan], grid_yy[within_nan]))
|
||||
n_pts = np.column_stack(
|
||||
(grid_xx[nan_mask], grid_yy[nan_mask]))
|
||||
try:
|
||||
from scipy.interpolate import griddata
|
||||
grid_content[within_nan] = griddata(
|
||||
v_pts, v_vals, n_pts, method='nearest'
|
||||
)
|
||||
grid_content[nan_mask] = griddata(
|
||||
v_pts, v_vals, n_pts, method='nearest')
|
||||
except Exception:
|
||||
grid_content[within_nan] = np.nanmean(grid_content[mask])
|
||||
grid_content[nan_mask] = np.nanmean(
|
||||
grid_content[valid_m])
|
||||
print(f"[掩膜填充] {np.sum(nan_mask)} 个 NaN → "
|
||||
f"最近邻填充完成")
|
||||
|
||||
# ★ 高斯平滑:消除 IDW 像素级斑点,仅在水体掩膜内
|
||||
if mask is not None:
|
||||
grid_content = self._gaussian_smooth_in_mask(
|
||||
grid_content, mask, sigma=1.2)
|
||||
else:
|
||||
# ── 原有完整流程(单图模式)──────────
|
||||
if shp_file is None:
|
||||
@ -3535,6 +3883,26 @@ if __name__ == "__main__":
|
||||
# 模块级函数:多进程 worker(必须在类外部定义,供 Pool.map 使用)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
class _KrigeChunk:
|
||||
"""克里金网格并行计算的数据载体"""
|
||||
__slots__ = ('grid_x', 'grid_y', 'n_closest')
|
||||
def __init__(self, grid_x, grid_y, n_closest):
|
||||
self.grid_x = grid_x
|
||||
self.grid_y = grid_y
|
||||
self.n_closest = n_closest
|
||||
|
||||
|
||||
def _krige_chunk_worker(ok_model, chunk):
|
||||
"""在子进程中执行单个网格切片的克里金预测"""
|
||||
import numpy as np
|
||||
z, _ = ok_model.execute(
|
||||
'grid', chunk.grid_x, chunk.grid_y,
|
||||
backend='loop',
|
||||
n_closest_points=chunk.n_closest,
|
||||
)
|
||||
return np.array(z)
|
||||
|
||||
|
||||
def _local_krige_block_worker(args):
|
||||
"""单个局部克里金块任务(独立进程入口,必须为模块级函数)"""
|
||||
(local_pts, local_vals, sub_grid_x, sub_grid_y,
|
||||
|
||||
@ -116,12 +116,14 @@ def rasterize_shp(shp_filepath, raster_fn_out, img_path, NoData_value=None):
|
||||
data_tmp = dataset_tmp.GetRasterBand(1).ReadAsArray()
|
||||
del dataset_tmp
|
||||
|
||||
# 创建和输入影像相同行列号、相同分辨率的水域掩膜,方便后续使用
|
||||
# 创建和输入影像相同行列号、相同分辨率的水域掩膜
|
||||
# ★ v2: 移除 imgdata_in == 0 的误杀逻辑
|
||||
# 高光谱影像第一波段(380nm)水体反射率接近0 → 大量水体像素被当成"影像外"跳过
|
||||
# 像素坐标越界检查 (coor_pixel bounds) 已能防止 SHP 超出影像范围的问题
|
||||
water_mask = np.zeros((im_height, im_width))
|
||||
for row in range(im_height):
|
||||
for column in range(im_width):
|
||||
coor = gdal.ApplyGeoTransform(geotransform, column, row)
|
||||
|
||||
coor_pixel = gdal.ApplyGeoTransform(inv_geotransform_tmp, coor[0], coor[1])
|
||||
coor_pixel = [int(num) for num in coor_pixel]
|
||||
|
||||
@ -130,9 +132,6 @@ def rasterize_shp(shp_filepath, raster_fn_out, img_path, NoData_value=None):
|
||||
if coor_pixel[1] < 0 or coor_pixel[1] >= data_tmp.shape[0]:
|
||||
continue
|
||||
|
||||
if imgdata_in[row, column] == 0: # 当shp区域比影像区域大时,略过
|
||||
continue
|
||||
|
||||
water_mask[row, column] = data_tmp[coor_pixel[1], coor_pixel[0]]
|
||||
|
||||
write_bands(img_path, raster_fn_out, water_mask)
|
||||
|
||||
@ -86,7 +86,8 @@ def get_wavelengths_from_bil_header(bil_file):
|
||||
|
||||
def get_spectral_sampling_points_chunked(bil_file, water_mask_shp, severe_glint=None, output_csvpath=None,
|
||||
interval=100, sample_radius=1, chunk_size=1000,
|
||||
use_adaptive_sampling=True, min_interval=50, max_interval=200):
|
||||
use_adaptive_sampling=True, min_interval=10, max_interval=200,
|
||||
water_ratio_threshold=0.35):
|
||||
"""
|
||||
基于bil文件、shp格式water_mask和severe_glint生成采样点并提取光谱数据(分块处理版本)
|
||||
|
||||
@ -225,14 +226,15 @@ def get_spectral_sampling_points_chunked(bil_file, water_mask_shp, severe_glint=
|
||||
local_y < sample_radius or local_y >= valid_chunk.shape[0] - sample_radius):
|
||||
return False
|
||||
|
||||
# 检查采样点周围区域是否全部有效
|
||||
# 检查采样点周围区域水体占比
|
||||
sample_area = valid_chunk[
|
||||
local_y - sample_radius:local_y + sample_radius + 1,
|
||||
x - sample_radius:x + sample_radius + 1
|
||||
]
|
||||
|
||||
# 如果采样区域内所有像元都有效
|
||||
if np.all(sample_area):
|
||||
# ★ v2: 允许窗口内部分非水体像素(窄水体友好,默认≥60%水体即通过)
|
||||
water_ratio = np.mean(sample_area.astype(np.float32))
|
||||
if water_ratio >= water_ratio_threshold:
|
||||
# 提取光谱数据(采样区域内的平均值)
|
||||
spectral_sample = []
|
||||
for band_idx in range(num_bands):
|
||||
@ -303,8 +305,8 @@ def get_spectral_sampling_points_chunked(bil_file, water_mask_shp, severe_glint=
|
||||
width_min_chunk = np.min(width_chunk_valid)
|
||||
width_max_chunk = np.max(width_chunk_valid)
|
||||
|
||||
# 使用基础间隔作为网格起点
|
||||
base_interval = min(interval, max_interval)
|
||||
# ★ v2: 基础步长用 min_interval(窄水体友好)
|
||||
base_interval = max(1, min(interval, min_interval))
|
||||
|
||||
# 使用网格化采样,但根据局部宽度调整间隔
|
||||
y = start_row
|
||||
@ -423,7 +425,8 @@ def get_spectral_sampling_points_chunked(bil_file, water_mask_shp, severe_glint=
|
||||
|
||||
def get_spectral_sampling_points(bil_file, water_mask_shp, severe_glint=None, output_csvpath=None,
|
||||
interval=100, sample_radius=1,
|
||||
use_adaptive_sampling=True, min_interval=50, max_interval=200):
|
||||
use_adaptive_sampling=True, min_interval=10, max_interval=200,
|
||||
water_ratio_threshold=0.35):
|
||||
"""
|
||||
基于bil文件、shp格式water_mask和severe_glint生成采样点并提取光谱数据
|
||||
|
||||
@ -568,12 +571,15 @@ def get_spectral_sampling_points(bil_file, water_mask_shp, severe_glint=None, ou
|
||||
y < sample_radius or y >= im_height - sample_radius):
|
||||
return False
|
||||
|
||||
# 检查采样点周围区域是否全部有效
|
||||
# 检查采样点周围区域水体占比
|
||||
sample_area = valid_area[y - sample_radius:y + sample_radius + 1,
|
||||
x - sample_radius:x + sample_radius + 1]
|
||||
|
||||
# 如果采样区域内所有像元都有效
|
||||
if np.all(sample_area):
|
||||
# ★ v2: 允许窗口内部分非水体像素(窄水体友好,默认≥60%水体即通过)
|
||||
water_ratio = np.mean(sample_area.astype(np.float32))
|
||||
if np.isnan(water_ratio):
|
||||
water_ratio = 0.0
|
||||
if water_ratio >= water_ratio_threshold:
|
||||
# 提取光谱数据(采样区域内的平均值)
|
||||
spectral_sample = []
|
||||
for band_idx in range(num_bands):
|
||||
@ -618,8 +624,8 @@ def get_spectral_sampling_points(bil_file, water_mask_shp, severe_glint=None, ou
|
||||
width_min = np.min(width_valid)
|
||||
width_max = np.max(width_valid)
|
||||
|
||||
# 使用基础间隔作为网格起点
|
||||
base_interval = min(interval, max_interval)
|
||||
# ★ v2: 基础步长用 min_interval(窄水体友好)
|
||||
base_interval = max(1, min(interval, min_interval))
|
||||
|
||||
# 使用网格化采样,但根据局部宽度调整间隔
|
||||
y = sample_radius
|
||||
|
||||
Reference in New Issue
Block a user