Step3 插值算法 OOM 修复 + 多进程加速 + 全链路累积改动(14 文件)

This commit is contained in:
DXC
2026-06-15 16:49:17 +08:00
parent 82e0b92af6
commit 60a9d7d922
14 changed files with 855 additions and 152 deletions

View File

@ -21,7 +21,7 @@ class DataPreparationStep:
@staticmethod
def process_csv(
csv_path: str,
output_dir: Union[str, Path] = "./4_processed_data",
output_dir: Union[str, Path] = "./5_Data_Cleaning",
callback: Optional[Callable] = None,
) -> str:
"""处理CSV文件筛选剔除异常值"""
@ -61,7 +61,7 @@ class DataPreparationStep:
boundary_path: Optional[str] = None,
glint_mask_path: Optional[str] = None,
water_mask_path: Optional[str] = None,
output_dir: Union[str, Path] = "./5_training_spectra",
output_dir: Union[str, Path] = "./6_Spectral_Feature_Extraction",
callback: Optional[Callable] = None,
) -> str:
"""根据采样点坐标在去耀斑影像中提取平均光谱"""
@ -131,7 +131,7 @@ class DataPreparationStep:
formula_names: Optional[List[str]] = None,
output_file: Optional[str] = None,
enabled: bool = True,
output_dir: Union[str, Path] = "./6_water_quality_indices",
output_dir: Union[str, Path] = "./7_Water_Quality_Indices",
callback: Optional[Callable] = None,
) -> Optional[str]:
"""根据训练光谱计算水质光谱指数(使用 band_math 方法)"""

View File

@ -135,7 +135,7 @@ class ModelingStep:
split_methods: Optional[List[str]] = None,
cv_folds: int = 5,
training_csv_path: Optional[str] = None,
output_dir: Union[str, Path] = "./7_Supervised_Model_Training",
output_dir: Union[str, Path] = "./8_Supervised_Model_Training",
callback: Optional[Callable] = None,
_report_generator=None,
) -> str:
@ -251,7 +251,7 @@ class ModelingStep:
if output_dir is not None:
non_empirical_dir = Path(output_dir)
else:
non_empirical_dir = Path.cwd() / "8_Regression_Modeling"
non_empirical_dir = Path.cwd() / "8_Non_Empirical_Regression"
non_empirical_dir.mkdir(parents=True, exist_ok=True)
if preprocessing_methods is None:
@ -430,7 +430,7 @@ def _apply_preprocessing_internal(
save_path = None
if preprocess_method == "SS":
models_dir = output_dir.parent.parent / "7_Supervised_Model_Training"
models_dir = output_dir.parent.parent / "8_Supervised_Model_Training"
models_dir.mkdir(parents=True, exist_ok=True)
save_path = str(models_dir / "scaler_params.pkl")
print(f"SS预处理: scaler模型将保存到 {save_path}")

View File

@ -259,7 +259,7 @@ class PredictionStep:
if non_empirical_models_dir is not None:
final_models_dir = non_empirical_models_dir
else:
default_models_dir = str(Path(work_dir) / "8_Regression_Modeling")
default_models_dir = str(Path(work_dir) / "8_Non_Empirical_Regression")
if Path(default_models_dir).exists():
final_models_dir = default_models_dir
else: