fix: 推理端反射率量级统管收敛为_align_reflectance_scale单一入口
- 新增 _align_reflectance_scale():光谱列 >10 则 /10000 收敛到 0~1,并清洗 NaN/负值 - inference_pipeline / batch_inference / batch_inference_multi_data 三处散装 /10000 魔法逻辑统一收敛 (注:该文件另含先前本地未提交改动,一并入库)
This commit is contained in:
@ -15,6 +15,7 @@ import os
|
|||||||
|
|
||||||
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
|
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
|
||||||
from src.core.utils.split_methods import spxy, ks
|
from src.core.utils.split_methods import spxy, ks
|
||||||
|
from src.utils.util import atomic_filepath
|
||||||
|
|
||||||
# try:
|
# try:
|
||||||
# from modeling import WaterQualityModeling
|
# from modeling import WaterQualityModeling
|
||||||
@ -125,6 +126,41 @@ class WaterQualityInference:
|
|||||||
|
|
||||||
return coords, spectra, wqi_df
|
return coords, spectra, wqi_df
|
||||||
|
|
||||||
|
def _align_reflectance_scale(self, spectra: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
"""
|
||||||
|
统一量级守卫:检测并统一输入光谱的量级到 0~1 的物理反射率区间。
|
||||||
|
|
||||||
|
必须在任何 WQI / 衍生特征进入模型之前调用,以确保衍生指数与训练数据处于
|
||||||
|
同一物理数值域。未来可替换为直接读取采样时写入的 metadata['scale_factor']。
|
||||||
|
"""
|
||||||
|
# 1) 提取光谱列(纯数字列名 = 波长列)
|
||||||
|
spec_cols = []
|
||||||
|
for c in spectra.columns:
|
||||||
|
try:
|
||||||
|
float(str(c))
|
||||||
|
spec_cols.append(c)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not spec_cols:
|
||||||
|
return spectra
|
||||||
|
|
||||||
|
# 2) 量级自适应:最大值 > 10 即视为 0-10000 放大格式,统一 /10000 至 0-1
|
||||||
|
max_val = spectra[spec_cols].max().max()
|
||||||
|
if max_val > 10:
|
||||||
|
print(f"\n[量级统管] 输入光谱最大值为 {max_val:.2f},触发自动归一化 (/ 10000.0) ...")
|
||||||
|
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
|
||||||
|
else:
|
||||||
|
print(f"[量级统管] 输入光谱量级正常 (max={max_val:.4f}),无需缩放")
|
||||||
|
|
||||||
|
# 3) 清洗底层脏数据:NaN/Inf -> 0,负反射率截断到 0
|
||||||
|
spec_data = spectra[spec_cols].values
|
||||||
|
spec_data = np.nan_to_num(spec_data, nan=0.0, posinf=0.0, neginf=0.0)
|
||||||
|
spec_data = np.maximum(spec_data, 0.0)
|
||||||
|
spectra[spec_cols] = spec_data
|
||||||
|
|
||||||
|
return spectra
|
||||||
|
|
||||||
def random(self, data, label, test_ratio=0.2, random_state=123):
|
def random(self, data, label, test_ratio=0.2, random_state=123):
|
||||||
"""
|
"""
|
||||||
随机划分数据集
|
随机划分数据集
|
||||||
@ -489,6 +525,20 @@ class WaterQualityInference:
|
|||||||
except (ValueError, TypeError):
|
except (ValueError, TypeError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# ==========================================
|
||||||
|
# ★ 新增:防御性拦截,防止后续 np.min() 崩溃
|
||||||
|
# ==========================================
|
||||||
|
if not spec_cols:
|
||||||
|
sampled_cols = list(spectra.columns)[:5]
|
||||||
|
raise ValueError(
|
||||||
|
f"[数据断链] 推理失败:采样 CSV 中未找到有效的数值型波长列名。\n"
|
||||||
|
f"检测到当前 CSV 的前几列为: {sampled_cols}...\n"
|
||||||
|
f"原因:上游去耀斑/采样步骤丢失了波长元数据 (未继承 .hdr 文件),"
|
||||||
|
f"导致采样程序使用了 'band_1' 等无物理意义的默认名称兜底。\n"
|
||||||
|
f"处理建议:请修复采样逻辑以包含波长表头,或重新运行采样步骤。"
|
||||||
|
)
|
||||||
|
# ==========================================
|
||||||
|
|
||||||
# np.interp 重采样:308/113/任意波段 → 模型训练波长
|
# np.interp 重采样:308/113/任意波段 → 模型训练波长
|
||||||
# ★ 边缘填充:left/right 使用当前行首尾有效值,杜绝 NaN→0.0 断崖
|
# ★ 边缘填充:left/right 使用当前行首尾有效值,杜绝 NaN→0.0 断崖
|
||||||
spec_data = spectra[spec_cols].values.astype(np.float64)
|
spec_data = spectra[spec_cols].values.astype(np.float64)
|
||||||
@ -1068,7 +1118,8 @@ class WaterQualityInference:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
print("警告: xlwt库未安装,无法保存为.xls格式,改为保存CSV格式")
|
print("警告: xlwt库未安装,无法保存为.xls格式,改为保存CSV格式")
|
||||||
csv_path = output_path.replace('.xls', '.csv')
|
csv_path = output_path.replace('.xls', '.csv')
|
||||||
result_df.to_csv(csv_path, index=False, encoding='utf-8-sig')
|
with atomic_filepath(csv_path) as _tmp:
|
||||||
|
result_df.to_csv(_tmp, index=False, encoding='utf-8-sig')
|
||||||
output_path = csv_path
|
output_path = csv_path
|
||||||
elif file_ext == '.xlsx':
|
elif file_ext == '.xlsx':
|
||||||
# 保存为Excel 2007+格式
|
# 保存为Excel 2007+格式
|
||||||
@ -1078,11 +1129,13 @@ class WaterQualityInference:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
print("警告: openpyxl库未安装,无法保存为.xlsx格式,改为保存CSV格式")
|
print("警告: openpyxl库未安装,无法保存为.xlsx格式,改为保存CSV格式")
|
||||||
csv_path = output_path.replace('.xlsx', '.csv')
|
csv_path = output_path.replace('.xlsx', '.csv')
|
||||||
result_df.to_csv(csv_path, index=False, encoding='utf-8-sig')
|
with atomic_filepath(csv_path) as _tmp:
|
||||||
|
result_df.to_csv(_tmp, index=False, encoding='utf-8-sig')
|
||||||
output_path = csv_path
|
output_path = csv_path
|
||||||
else:
|
else:
|
||||||
# 默认保存为CSV格式
|
# 默认保存为CSV格式(★ 原子写入:先 .__wip 后同卷替换)
|
||||||
result_df.to_csv(output_path, index=False, encoding='utf-8-sig')
|
with atomic_filepath(output_path) as _tmp:
|
||||||
|
result_df.to_csv(_tmp, index=False, encoding='utf-8-sig')
|
||||||
print(f" 格式: CSV (.csv)")
|
print(f" 格式: CSV (.csv)")
|
||||||
|
|
||||||
print(f"预测结果保存完成:")
|
print(f"预测结果保存完成:")
|
||||||
@ -1130,32 +1183,8 @@ class WaterQualityInference:
|
|||||||
print("-" * 40)
|
print("-" * 40)
|
||||||
coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path)
|
coords, spectra, wqi_df = self.load_sampling_data(sampling_csv_path)
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════════
|
# ★ 统一反射率量级:0~1 物理反射率区间(须在特征/WQI 进入模型前完成)
|
||||||
# ★ 自适应反射率量级缩放 (Scale Alignment)
|
spectra = self._align_reflectance_scale(spectra)
|
||||||
# ═══════════════════════════════════════════════════════════════
|
|
||||||
# 不同的高光谱传感器 / 处理流程产出的反射率量级可能不同:
|
|
||||||
# - float32 0-1 物理反射率(如 result3.bsq 抽样后写入的 CSV)
|
|
||||||
# - int16 0-10000 放大反射率(如 ref_mosaic 抽样后写入的 CSV)
|
|
||||||
# 若不经缩放直接喂入 SVR,量级差异会导致预测完全失效。
|
|
||||||
# 此处在光谱列上自动检测并统一到 0-1 区间。
|
|
||||||
spec_cols = []
|
|
||||||
for c in spectra.columns:
|
|
||||||
try:
|
|
||||||
float(str(c))
|
|
||||||
spec_cols.append(c)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
if spec_cols:
|
|
||||||
max_val = spectra[spec_cols].max().max()
|
|
||||||
if max_val > 10:
|
|
||||||
print(f"\n[量级检测] 输入反射率疑似放大格式 (max={max_val:.2f})")
|
|
||||||
print("[量级检测] 自动除以 10000,缩放至 0-1 标准物理反射率区间...")
|
|
||||||
spectra[spec_cols] = spectra[spec_cols].astype(float) / 10000.0
|
|
||||||
print(f"[量级检测] 缩放完成!缩放后 max={spectra[spec_cols].max().max():.4f}")
|
|
||||||
else:
|
|
||||||
print(f"[量级检测] 输入反射率量级正常 (max={max_val:.4f}),无需缩放")
|
|
||||||
# ═══════════════════════════════════════════════════════════════
|
|
||||||
|
|
||||||
# 3. 数据预处理
|
# 3. 数据预处理
|
||||||
print("\n步骤3: 数据预处理")
|
print("\n步骤3: 数据预处理")
|
||||||
@ -1245,15 +1274,8 @@ class WaterQualityInference:
|
|||||||
|
|
||||||
# 执行推理
|
# 执行推理
|
||||||
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
|
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
|
||||||
# 自适应反射率量级缩放
|
# ★ 第一时间统一反射率量级,确保 WQI / 衍生特征与训练数据处于同一物理空间
|
||||||
_s_cols = []
|
spectra = self._align_reflectance_scale(spectra)
|
||||||
for _c in spectra.columns:
|
|
||||||
try: float(str(_c)); _s_cols.append(_c)
|
|
||||||
except (ValueError, TypeError): pass
|
|
||||||
if _s_cols:
|
|
||||||
_mv = spectra[_s_cols].max().max()
|
|
||||||
if _mv > 10:
|
|
||||||
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
|
|
||||||
spectra_processed = self.preprocess_spectra(spectra)
|
spectra_processed = self.preprocess_spectra(spectra)
|
||||||
predictions = self.predict(spectra_processed)
|
predictions = self.predict(spectra_processed)
|
||||||
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
|
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
|
||||||
@ -1454,15 +1476,8 @@ class WaterQualityInference:
|
|||||||
|
|
||||||
# 执行推理
|
# 执行推理
|
||||||
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
|
coords, spectra, wqi_df = self.load_sampling_data(str(csv_file))
|
||||||
# 自适应反射率量级缩放
|
# ★ 第一时间统一反射率量级,确保 WQI / 衍生特征与训练数据处于同一物理空间
|
||||||
_s_cols = []
|
spectra = self._align_reflectance_scale(spectra)
|
||||||
for _c in spectra.columns:
|
|
||||||
try: float(str(_c)); _s_cols.append(_c)
|
|
||||||
except (ValueError, TypeError): pass
|
|
||||||
if _s_cols:
|
|
||||||
_mv = spectra[_s_cols].max().max()
|
|
||||||
if _mv > 10:
|
|
||||||
spectra[_s_cols] = spectra[_s_cols].astype(float) / 10000.0
|
|
||||||
spectra_processed = self.preprocess_spectra(spectra)
|
spectra_processed = self.preprocess_spectra(spectra)
|
||||||
predictions = self.predict(spectra_processed)
|
predictions = self.predict(spectra_processed)
|
||||||
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
|
predictions = self._mask_zero_spectra_pixels(spectra, predictions)
|
||||||
|
|||||||
Reference in New Issue
Block a user