fix: 推理端反射率量级统管收敛为_align_reflectance_scale单一入口

- 新增 _align_reflectance_scale():光谱列 >10 则 /10000 收敛到 0~1,并清洗 NaN/负值
- inference_pipeline / batch_inference / batch_inference_multi_data 三处散装 /10000 魔法逻辑统一收敛
(注:该文件另含先前本地未提交改动,一并入库)
This commit is contained in:
duxin
2026-09-08 17:31:51 +08:00
parent 81ae400294
commit 8f416c728b

View File

@ -15,6 +15,7 @@ import os
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
from src.core.utils.split_methods import spxy, ks from src.core.utils.split_methods import spxy, ks
from src.utils.util import atomic_filepath
# try: # try:
# from modeling import WaterQualityModeling # from modeling import WaterQualityModeling
@ -125,6 +126,41 @@ class WaterQualityInference:
return coords, spectra, wqi_df return coords, spectra, wqi_df
def _align_reflectance_scale(self, spectra: pd.DataFrame) -> pd.DataFrame:
"""
统一量级守卫:检测并统一输入光谱的量级到 0~1 的物理反射率区间。
必须在任何 WQI / 衍生特征进入模型之前调用,以确保衍生指数与训练数据处于
同一物理数值域。未来可替换为直接读取采样时写入的 metadata['scale_factor']。
"""
# 1) 提取光谱列(纯数字列名 = 波长列)
spec_cols = []
for c in spectra.columns:
try:
float(str(c))
spec_cols.append(c)
except (ValueError, TypeError):
pass
if not spec_cols:
return spectra
# 2) 量级自适应:最大值 > 10 即视为 0-10000 放大格式,统一 /10000 至 0-1
max_val = spectra[spec_cols].max().max()
if max_val > 10:
print(f"\n[量级统管] 输入光谱最大值为 {max_val:.2f},触发自动归一化 (/ 10000.0) ...")
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
else:
print(f"[量级统管] 输入光谱量级正常 (max={max_val:.4f}),无需缩放")
# 3) 清洗底层脏数据:NaN/Inf -> 0,负反射率截断到 0
spec_data = spectra[spec_cols].values
spec_data = np.nan_to_num(spec_data, nan=0.0, posinf=0.0, neginf=0.0)
spec_data = np.maximum(spec_data, 0.0)
spectra[spec_cols] = spec_data
return spectra
def random(self, data, label, test_ratio=0.2, random_state=123): def random(self, data, label, test_ratio=0.2, random_state=123):
""" """
随机划分数据集 随机划分数据集
@ -489,6 +525,20 @@ class WaterQualityInference:
except (ValueError, TypeError): except (ValueError, TypeError):
pass pass
# ==========================================
# ★ 新增:防御性拦截,防止后续 np.min() 崩溃
# ==========================================
if not spec_cols:
sampled_cols = list(spectra.columns)[:5]
raise ValueError(
f"[数据断链] 推理失败:采样 CSV 中未找到有效的数值型波长列名。\n"
f"检测到当前 CSV 的前几列为: {sampled_cols}...\n"
f"原因:上游去耀斑/采样步骤丢失了波长元数据 (未继承 .hdr 文件),"
f"导致采样程序使用了 'band_1' 等无物理意义的默认名称兜底。\n"
f"处理建议:请修复采样逻辑以包含波长表头,或重新运行采样步骤。"
)
# ==========================================
# np.interp 重采样:308/113/任意波段 → 模型训练波长 # np.interp 重采样:308/113/任意波段 → 模型训练波长
# ★ 边缘填充:left/right 使用当前行首尾有效值,杜绝 NaN→0.0 断崖 # ★ 边缘填充:left/right 使用当前行首尾有效值,杜绝 NaN→0.0 断崖
spec_data = spectra[spec_cols].values.astype(np.float64) spec_data = spectra[spec_cols].values.astype(np.float64)
@ -1068,7 +1118,8 @@ class WaterQualityInference:
except ImportError: except ImportError:
print("警告: xlwt库未安装,无法保存为.xls格式,改为保存CSV格式") print("警告: xlwt库未安装,无法保存为.xls格式,改为保存CSV格式")
csv_path = output_path.replace('.xls', '.csv') csv_path = output_path.replace('.xls', '.csv')
result_df.to_csv(csv_path, index=False, encoding='utf-8-sig') with atomic_filepath(csv_path) as _tmp:
result_df.to_csv(_tmp, index=False, encoding='utf-8-sig')
output_path = csv_path output_path = csv_path
elif file_ext == '.xlsx': elif file_ext == '.xlsx':
# 保存为Excel 2007+格式 # 保存为Excel 2007+格式
@ -1078,11 +1129,13 @@ class WaterQualityInference:
except ImportError: except ImportError:
print("警告: openpyxl库未安装,无法保存为.xlsx格式,改为保存CSV格式") print("警告: openpyxl库未安装,无法保存为.xlsx格式,改为保存CSV格式")
csv_path = output_path.replace('.xlsx', '.csv') csv_path = output_path.replace('.xlsx', '.csv')
result_df.to_csv(csv_path, index=False, encoding='utf-8-sig') with atomic_filepath(csv_path) as _tmp:
result_df.to_csv(_tmp, index=False, encoding='utf-8-sig')
output_path = csv_path output_path = csv_path
else: else:
# 默认保存为CSV格式 # 默认保存为CSV格式(★ 原子写入:先 .__wip 后同卷替换)
result_df.to_csv(output_path, index=False, encoding='utf-8-sig') with atomic_filepath(output_path) as _tmp:
result_df.to_csv(_tmp, index=False, encoding='utf-8-sig')
print(f" 格式: CSV (.csv)") print(f" 格式: CSV (.csv)")
print(f"预测结果保存完成:") print(f"预测结果保存完成:")
@ -1130,32 +1183,8 @@ class WaterQualityInference:
print("-" * 40) print("-" * 40)
coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path) coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path)
# ═══════════════════════════════════════════════════════════════ # ★ 统一反射率量级:0~1 物理反射率区间(须在特征/WQI 进入模型前完成)
# ★ 自适应反射率量级缩放 (Scale Alignment) spectra = self._align_reflectance_scale(spectra)
# ═══════════════════════════════════════════════════════════════
# 不同的高光谱传感器 / 处理流程产出的反射率量级可能不同:
# - float32 0-1 物理反射率(如 result3.bsq 抽样后写入的 CSV)
# - int16 0-10000 放大反射率(如 ref_mosaic 抽样后写入的 CSV)
# 若不经缩放直接喂入 SVR,量级差异会导致预测完全失效。
# 此处在光谱列上自动检测并统一到 0-1 区间。
spec_cols = []
for c in spectra.columns:
try:
float(str(c))
spec_cols.append(c)
except (ValueError, TypeError):
pass
if spec_cols:
max_val = spectra[spec_cols].max().max()
if max_val > 10:
print(f"\n[量级检测] 输入反射率疑似放大格式 (max={max_val:.2f})")
print("[量级检测] 自动除以 10000,缩放至 0-1 标准物理反射率区间...")
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
print(f"[量级检测] 缩放完成!缩放后 max={spectra[spec_cols].max().max():.4f}")
else:
print(f"[量级检测] 输入反射率量级正常 (max={max_val:.4f}),无需缩放")
# ═══════════════════════════════════════════════════════════════
# 3. 数据预处理 # 3. 数据预处理
print("\n步骤3: 数据预处理") print("\n步骤3: 数据预处理")
@ -1245,15 +1274,8 @@ class WaterQualityInference:
# 执行推理 # 执行推理
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file)) coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
# 自适应反射率量级缩放 # ★ 第一时间统一反射率量级,确保 WQI / 衍生特征与训练数据处于同一物理空间
_s_cols = [] spectra = self._align_reflectance_scale(spectra)
for _c in spectra.columns:
try: float(str(_c)); _s_cols.append(_c)
except (ValueError, TypeError): pass
if _s_cols:
_mv = spectra[_s_cols].max().max()
if _mv > 10:
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
spectra_processed = self.preprocess_spectra(spectra) spectra_processed = self.preprocess_spectra(spectra)
predictions = self.predict(spectra_processed) predictions = self.predict(spectra_processed)
predictions = self._mask_zero_spectra_pixels(spectra, predictions) predictions = self._mask_zero_spectra_pixels(spectra, predictions)
@ -1454,15 +1476,8 @@ class WaterQualityInference:
# 执行推理 # 执行推理
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file)) coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
# 自适应反射率量级缩放 # ★ 第一时间统一反射率量级,确保 WQI / 衍生特征与训练数据处于同一物理空间
_s_cols = [] spectra = self._align_reflectance_scale(spectra)
for _c in spectra.columns:
try: float(str(_c)); _s_cols.append(_c)
except (ValueError, TypeError): pass
if _s_cols:
_mv = spectra[_s_cols].max().max()
if _mv > 10:
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
spectra_processed = self.preprocess_spectra(spectra) spectra_processed = self.preprocess_spectra(spectra)
predictions = self.predict(spectra_processed) predictions = self.predict(spectra_processed)
predictions = self._mask_zero_spectra_pixels(spectra, predictions) predictions = self._mask_zero_spectra_pixels(spectra, predictions)