feat: 训练框架适配 DualStream_MNF + 部署矩阵 .npz 导出

modeling_batch.py:
- preprocessing_methods 新增 DualStream_MNF
- save_model 新增 C++/Rust 部署导出逻辑:
  自动检测 Pipeline 中的 MNFTransformer + SVR,
  提取 mean_/W_mnf_/dual_coef_/support_vectors_/
  intercept_/gamma/kernel 等纯量矩阵,
  保存为 *_deploy.npz 文件
- 新增 _find_mnf_in_pipeline / _find_svr_in_pipeline /
  _find_physical_extractor_in_pipeline 辅助方法,
  支持 FeatureUnion 嵌套递归查找
This commit is contained in:
duxin
2026-07-29 10:02:37 +08:00
parent 7d8d848e58
commit 4347988ab2

View File

@ -223,7 +223,8 @@ class WaterQualityModelingBatch:
# 预处理方法列表
self.preprocessing_methods = [
"None", "MMS", "SS", "CT", "SNV", "MA", "SG", "MSC", "D1", "D2", "DT", "WVAE"
"None", "MMS", "SS", "CT", "SNV", "MA", "SG", "MSC", "D1", "D2", "DT", "WVAE",
"DualStream_MNF"
]
# 样本划分方法列表
@ -716,7 +717,7 @@ class WaterQualityModelingBatch:
"""
# 清理目标列名,移除可能的特殊字符
safe_target_name = "".join(c for c in target_column_name if c.isalnum() or c in ('-', '_')).rstrip()
filename = f"{safe_target_name}_{preprocess_method}_{model_name}.joblib"
filepath = self.artifacts_dir / filename
@ -732,6 +733,112 @@ class WaterQualityModelingBatch:
joblib.dump(save_data, filepath)
print(f"模型已保存: {filepath}")
# ═══════════════════════════════════════════════════════════
# ★ C++/Rust 部署导出:提取 MNF + SVR 纯量矩阵
# ═══════════════════════════════════════════════════════════
if not isinstance(model, Pipeline):
return
_mnf = self._find_mnf_in_pipeline(model)
_svr = self._find_svr_in_pipeline(model)
_phys = self._find_physical_extractor_in_pipeline(model)
if _mnf is not None and _svr is not None:
_deploy = {
'mnf_mean': np.asarray(_mnf.mean_).ravel(),
'mnf_W': np.asarray(_mnf.W_mnf_),
'mnf_n_components': int(_mnf.n_components),
}
# SVR 部署矩阵(rbf kernel 需要 support_vectors_)
_deploy['svr_dual_coef'] = np.asarray(_svr.dual_coef_)
_deploy['svr_intercept'] = np.asarray(_svr.intercept_)
if hasattr(_svr, 'support_vectors_'):
_deploy['svr_support_vectors'] = np.asarray(_svr.support_vectors_)
_deploy['svr_gamma'] = float(getattr(_svr, '_gamma', 1.0))
# kernel type for C++ dispatch
_kernel = str(getattr(_svr, 'kernel', 'rbf')).lower()
_deploy['svr_kernel'] = _kernel
if _kernel == 'poly':
_deploy['svr_degree'] = int(getattr(_svr, 'degree', 3))
_deploy['svr_coef0'] = float(getattr(_svr, 'coef0', 0.0))
# 物理特征索引(C++ 端据此定位波段列)
if _phys is not None:
_deploy['physical_wavelengths'] = np.asarray(
_phys.wavelengths, dtype=np.float64
)
_feat_map = {}
for name, ia, ib in _phys._feat_cols_:
_feat_map[name] = {
'wl_a': float(_phys.wavelengths[ia]),
'wl_b': float(_phys.wavelengths[ib]),
'idx_a': int(ia),
'idx_b': int(ib),
}
_deploy['physical_feature_map'] = _feat_map
npz_path = str(filepath).replace('.joblib', '_deploy.npz')
np.savez_compressed(npz_path, **_deploy)
print(f"[部署导出] MNF+SVR 纯量矩阵已保存: {npz_path}")
elif _mnf is None and _svr is not None:
print("[部署导出] 仅检测到 SVR 未检测到 MNFTransformer,跳过 .npz 导出")
# ═══════════════════════════════════════════════════════════
# 部署矩阵提取辅助方法:从 Pipeline 中定位特定 Transformer
# ═══════════════════════════════════════════════════════════
@staticmethod
def _find_transformer_in_pipeline(pipeline, target_class_name):
"""在 Pipeline 的 'preproc' 步骤中递归查找指定类型的 Transformer。
支持 FeatureUnion 嵌套:若 preproc 是 FeatureUnion,
则遍历其 transformer_list 逐个子变压器查找。
"""
from sklearn.pipeline import FeatureUnion
preproc = pipeline.named_steps.get('preproc')
if preproc is None:
return None
if isinstance(preproc, FeatureUnion):
for name, trans in preproc.transformer_list:
found = WaterQualityModelingBatch._find_transformer_in_step(
trans, target_class_name)
if found is not None:
return found
return None
return WaterQualityModelingBatch._find_transformer_in_step(
preproc, target_class_name)
@staticmethod
def _find_transformer_in_step(step, target_class_name):
"""在单个 Pipeline 步骤(可能是 Pipeline 自身)中查找 Transformer。"""
if step.__class__.__name__ == target_class_name:
return step
# 处理 Pipeline 嵌套
if hasattr(step, 'named_steps'):
for sub_name, sub_step in step.named_steps.items():
found = WaterQualityModelingBatch._find_transformer_in_step(
sub_step, target_class_name)
if found is not None:
return found
return None
@staticmethod
def _find_mnf_in_pipeline(pipeline):
return WaterQualityModelingBatch._find_transformer_in_pipeline(
pipeline, 'MNFTransformer')
@staticmethod
def _find_svr_in_pipeline(pipeline):
model_step = pipeline.named_steps.get('model')
if model_step is not None and model_step.__class__.__name__ == 'SVR':
return model_step
return None
@staticmethod
def _find_physical_extractor_in_pipeline(pipeline):
return WaterQualityModelingBatch._find_transformer_in_pipeline(
pipeline, 'PhysicalFeatureExtractor')
def train_models_batch(self, csv_path: str, feature_start_column: Union[int, str],
preprocessing_methods: Union[str, List[str]] = "None",
model_names: Union[str, List[str]] = "RF",