From 4347988ab2c9b33a20379b5829b0868e3cdb2e6c Mon Sep 17 00:00:00 2001 From: duxin Date: Wed, 29 Jul 2026 10:02:37 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=AE=AD=E7=BB=83=E6=A1=86=E6=9E=B6?= =?UTF-8?q?=E9=80=82=E9=85=8D=20DualStream=5FMNF=20+=20=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E7=9F=A9=E9=98=B5=20.npz=20=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit modeling_batch.py: - preprocessing_methods 新增 DualStream_MNF - save_model 新增 C++/Rust 部署导出逻辑: 自动检测 Pipeline 中的 MNFTransformer + SVR， 提取 mean_/W_mnf_/dual_coef_/support_vectors_/ intercept_/gamma/kernel 等纯量矩阵， 保存为 *_deploy.npz 文件 - 新增 _find_mnf_in_pipeline / _find_svr_in_pipeline / _find_physical_extractor_in_pipeline 辅助方法， 支持 FeatureUnion 嵌套递归查找 --- src/core/modeling/modeling_batch.py | 111 +++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/src/core/modeling/modeling_batch.py b/src/core/modeling/modeling_batch.py index 58e5b86..fe81af0 100644 --- a/src/core/modeling/modeling_batch.py +++ b/src/core/modeling/modeling_batch.py @@ -223,7 +223,8 @@ class WaterQualityModelingBatch: # 预处理方法列表 self.preprocessing_methods = [ - "None", "MMS", "SS", "CT", "SNV", "MA", "SG", "MSC", "D1", "D2", "DT", "WVAE" + "None", "MMS", "SS", "CT", "SNV", "MA", "SG", "MSC", "D1", "D2", "DT", "WVAE", + "DualStream_MNF" ] # 样本划分方法列表 @@ -716,7 +717,7 @@ class WaterQualityModelingBatch: """ # 清理目标列名，移除可能的特殊字符 safe_target_name = "".join(c for c in target_column_name if c.isalnum() or c in ('-', '_')).rstrip() - + filename = f"{safe_target_name}_{preprocess_method}_{model_name}.joblib" filepath = self.artifacts_dir / filename @@ -732,6 +733,112 @@ class WaterQualityModelingBatch: joblib.dump(save_data, filepath) print(f"模型已保存: {filepath}") + # ═══════════════════════════════════════════════════════════ + # ★ C++/Rust 部署导出：提取 MNF + SVR 纯量矩阵 + # ═══════════════════════════════════════════════════════════ + if not isinstance(model, Pipeline): + return + + _mnf = self._find_mnf_in_pipeline(model) + _svr = self._find_svr_in_pipeline(model) + _phys = self._find_physical_extractor_in_pipeline(model) + + if _mnf is not None and _svr is not None: + _deploy = { + 'mnf_mean': np.asarray(_mnf.mean_).ravel(), + 'mnf_W': np.asarray(_mnf.W_mnf_), + 'mnf_n_components': int(_mnf.n_components), + } + # SVR 部署矩阵（rbf kernel 需要 support_vectors_） + _deploy['svr_dual_coef'] = np.asarray(_svr.dual_coef_) + _deploy['svr_intercept'] = np.asarray(_svr.intercept_) + if hasattr(_svr, 'support_vectors_'): + _deploy['svr_support_vectors'] = np.asarray(_svr.support_vectors_) + _deploy['svr_gamma'] = float(getattr(_svr, '_gamma', 1.0)) + # kernel type for C++ dispatch + _kernel = str(getattr(_svr, 'kernel', 'rbf')).lower() + _deploy['svr_kernel'] = _kernel + if _kernel == 'poly': + _deploy['svr_degree'] = int(getattr(_svr, 'degree', 3)) + _deploy['svr_coef0'] = float(getattr(_svr, 'coef0', 0.0)) + + # 物理特征索引（C++ 端据此定位波段列） + if _phys is not None: + _deploy['physical_wavelengths'] = np.asarray( + _phys.wavelengths, dtype=np.float64 + ) + _feat_map = {} + for name, ia, ib in _phys._feat_cols_: + _feat_map[name] = { + 'wl_a': float(_phys.wavelengths[ia]), + 'wl_b': float(_phys.wavelengths[ib]), + 'idx_a': int(ia), + 'idx_b': int(ib), + } + _deploy['physical_feature_map'] = _feat_map + + npz_path = str(filepath).replace('.joblib', '_deploy.npz') + np.savez_compressed(npz_path, **_deploy) + print(f"[部署导出] MNF+SVR 纯量矩阵已保存: {npz_path}") + elif _mnf is None and _svr is not None: + print("[部署导出] 仅检测到 SVR 未检测到 MNFTransformer，跳过 .npz 导出") + + # ═══════════════════════════════════════════════════════════ + # 部署矩阵提取辅助方法：从 Pipeline 中定位特定 Transformer + # ═══════════════════════════════════════════════════════════ + + @staticmethod + def _find_transformer_in_pipeline(pipeline, target_class_name): + """在 Pipeline 的 'preproc' 步骤中递归查找指定类型的 Transformer。 + + 支持 FeatureUnion 嵌套：若 preproc 是 FeatureUnion， + 则遍历其 transformer_list 逐个子变压器查找。 + """ + from sklearn.pipeline import FeatureUnion + preproc = pipeline.named_steps.get('preproc') + if preproc is None: + return None + if isinstance(preproc, FeatureUnion): + for name, trans in preproc.transformer_list: + found = WaterQualityModelingBatch._find_transformer_in_step( + trans, target_class_name) + if found is not None: + return found + return None + return WaterQualityModelingBatch._find_transformer_in_step( + preproc, target_class_name) + + @staticmethod + def _find_transformer_in_step(step, target_class_name): + """在单个 Pipeline 步骤（可能是 Pipeline 自身）中查找 Transformer。""" + if step.__class__.__name__ == target_class_name: + return step + # 处理 Pipeline 嵌套 + if hasattr(step, 'named_steps'): + for sub_name, sub_step in step.named_steps.items(): + found = WaterQualityModelingBatch._find_transformer_in_step( + sub_step, target_class_name) + if found is not None: + return found + return None + + @staticmethod + def _find_mnf_in_pipeline(pipeline): + return WaterQualityModelingBatch._find_transformer_in_pipeline( + pipeline, 'MNFTransformer') + + @staticmethod + def _find_svr_in_pipeline(pipeline): + model_step = pipeline.named_steps.get('model') + if model_step is not None and model_step.__class__.__name__ == 'SVR': + return model_step + return None + + @staticmethod + def _find_physical_extractor_in_pipeline(pipeline): + return WaterQualityModelingBatch._find_transformer_in_pipeline( + pipeline, 'PhysicalFeatureExtractor') + def train_models_batch(self, csv_path: str, feature_start_column: Union[int, str], preprocessing_methods: Union[str, List[str]] = "None", model_names: Union[str, List[str]] = "RF",