全局修正
This commit is contained in:
@ -19,6 +19,7 @@ from sklearn.cross_decomposition import PLSRegression
|
||||
from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor, ExtraTreesRegressor
|
||||
from sklearn.tree import DecisionTreeRegressor
|
||||
from sklearn.neural_network import MLPRegressor
|
||||
from sklearn.pipeline import Pipeline
|
||||
from joblib import parallel_backend
|
||||
# 第三方模型导入
|
||||
# try:
|
||||
@ -44,7 +45,7 @@ import os
|
||||
is_frozen_env = getattr(sys, 'frozen', False)
|
||||
safe_n_jobs = 1 if is_frozen_env else -1
|
||||
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
|
||||
from src.core.utils.split_methods import spxy, ks
|
||||
|
||||
|
||||
@ -454,25 +455,28 @@ class WaterQualityModelingBatch:
|
||||
else:
|
||||
raise ValueError(f"不支持的划分方法: {method}. 支持的方法: {self.split_methods}")
|
||||
|
||||
def train_single_model(self, X: np.ndarray, y: pd.Series, model_name: str,
|
||||
def train_single_model(self, X_raw: pd.DataFrame, y: pd.Series, model_name: str,
|
||||
cv_folds: int = 5, scoring: str = 'neg_mean_squared_error',
|
||||
test_size: float = 0.2, random_state: int = 42,
|
||||
split_method: str = "random") -> Dict:
|
||||
split_method: str = "random",
|
||||
preprocess_method: str = "None") -> Dict:
|
||||
"""
|
||||
训练单个回归模型
|
||||
训练单个回归模型（Pipeline 化：preprocess_method 字符串内部构造 Pipeline，
|
||||
scaler/MSC.mean_spectrum_ 等状态被绑定在 best_model 上，CV 与 test 评估
|
||||
都在「只拟合训练 fold 的 scaler」之上，避免传统「X_full → scaler.fit →
|
||||
split → CV」造成的数据泄露）。
|
||||
|
||||
Args:
|
||||
X: 特征数据
|
||||
X_raw: 原始特征数据（未经预处理，DataFrame 形态方便 Pipeline 内部转换）
|
||||
y: 目标值数据
|
||||
model_name: 模型名称
|
||||
cv_folds: 交叉验证折数
|
||||
scoring: 评分指标
|
||||
test_size: 测试集比例
|
||||
random_state: 随机种子
|
||||
split_method: 数据划分方法
|
||||
preprocess_method: 预处理方法字符串（如 'None' / 'SS' / 'MSC'）；
|
||||
训练结束后 best_model 字段即为 sklearn Pipeline。
|
||||
其余参数（cv_folds / scoring / test_size / random_state / split_method）
|
||||
含义保持不变。
|
||||
|
||||
Returns:
|
||||
训练结果字典
|
||||
训练结果字典；'model' 字段现在是 sklearn.pipeline.Pipeline（含 scaler）
|
||||
"""
|
||||
if model_name not in self.model_configs:
|
||||
raise ValueError(f"不支持的模型: {model_name}")
|
||||
@ -483,18 +487,18 @@ class WaterQualityModelingBatch:
|
||||
print(f"模型 {model_name} 不可用，请安装相应的库")
|
||||
return None
|
||||
|
||||
print(f"开始训练模型: {model_name}")
|
||||
print(f"开始训练模型: {model_name} (预处理: {preprocess_method})")
|
||||
|
||||
# 使用指定方法分割训练集和测试集
|
||||
# 使用指定方法分割训练集和测试集（用原始 X_raw，Pipeline 内置 transform 处理）
|
||||
X_train, X_test, y_train, y_test = self.split_data(
|
||||
X, y, method=split_method, test_size=test_size, random_state=random_state
|
||||
X_raw, y, method=split_method, test_size=test_size, random_state=random_state
|
||||
)
|
||||
|
||||
print(f"数据分割完成:")
|
||||
print(f" 训练集样本数: {X_train.shape[0]}")
|
||||
print(f" 测试集样本数: {X_test.shape[0]}")
|
||||
|
||||
# 创建模型实例
|
||||
# 构造 base_model
|
||||
if callable(config['model']):
|
||||
base_model = config['model']()
|
||||
else:
|
||||
@ -506,12 +510,25 @@ class WaterQualityModelingBatch:
|
||||
elif model_name == 'LightGBM':
|
||||
base_model.set_params(verbose=-1)
|
||||
|
||||
# 随机搜索 —— 替代穷举式 GridSearchCV，大幅降低寻优时间
|
||||
# ============ 关键：把预处理器塞进 Pipeline ============
|
||||
preproc = get_preprocessing_transformer(preprocess_method)
|
||||
pipeline = Pipeline([
|
||||
('preproc', preproc),
|
||||
('model', base_model),
|
||||
])
|
||||
|
||||
# RandomizedSearchCV 需要以「步骤名__参数名」的格式索引参数网格；
|
||||
# 我们原有的 config['params'] 是模型层的（无 __），统一加 model__ 前缀。
|
||||
prefixed_params = {
|
||||
f"model__{k}": v for k, v in config['params'].items()
|
||||
}
|
||||
|
||||
# 随机搜索：直接对 Pipeline 调优（scaler 仅在 train fold 上 fit）
|
||||
cv_strategy = KFold(n_splits=cv_folds, shuffle=True, random_state=random_state)
|
||||
|
||||
grid_search = RandomizedSearchCV(
|
||||
base_model,
|
||||
config['params'],
|
||||
pipeline,
|
||||
prefixed_params,
|
||||
n_iter=10,
|
||||
cv=cv_strategy,
|
||||
scoring=scoring,
|
||||
@ -522,20 +539,22 @@ class WaterQualityModelingBatch:
|
||||
|
||||
grid_search.fit(X_train, y_train)
|
||||
|
||||
# 获取最佳模型
|
||||
# 获取最佳模型（已是 Pipeline）
|
||||
best_model = grid_search.best_estimator_
|
||||
|
||||
# 交叉验证评估（在训练集上）
|
||||
cv_scores = cross_val_score(best_model, X_train, y_train, cv=cv_strategy, scoring=scoring)
|
||||
# 交叉验证评估（在训练集上）：cross_val_score 会对 Pipeline 重 clone，
|
||||
# 保证每个 fold 重 fit 预处理，CV 评分反映「无泄露」真实泛化能力
|
||||
cv_scores = cross_val_score(best_model, X_train, y_train, cv=cv_strategy,
|
||||
scoring=scoring, n_jobs=safe_n_jobs)
|
||||
|
||||
# 计算训练集上的回归指标
|
||||
# 计算训练集上的回归指标（Pipeline 内 fit_transform 只发生一次，已 fit 完毕）
|
||||
y_train_pred = best_model.predict(X_train)
|
||||
train_mse = mean_squared_error(y_train, y_train_pred)
|
||||
train_mae = mean_absolute_error(y_train, y_train_pred)
|
||||
train_r2 = r2_score(y_train, y_train_pred)
|
||||
train_rmse = np.sqrt(train_mse)
|
||||
|
||||
# 计算测试集上的回归指标
|
||||
# 计算测试集上的回归指标（用训练集 fit 出的 scaler，正确的 deploy-time 行为）
|
||||
y_test_pred = best_model.predict(X_test)
|
||||
test_mse = mean_squared_error(y_test, y_test_pred)
|
||||
test_mae = mean_absolute_error(y_test, y_test_pred)
|
||||
@ -562,7 +581,10 @@ class WaterQualityModelingBatch:
|
||||
# 数据分割信息
|
||||
'train_size': X_train.shape[0],
|
||||
'test_size': X_test.shape[0],
|
||||
'split_method': split_method
|
||||
'split_method': split_method,
|
||||
# Pipeline 信息（用于诊断 / metadata）
|
||||
'preprocess_method': preprocess_method,
|
||||
'is_pipeline': isinstance(best_model, Pipeline),
|
||||
}
|
||||
|
||||
print(f"模型 {model_name} 训练完成:")
|
||||
@ -714,21 +736,21 @@ class WaterQualityModelingBatch:
|
||||
print(f"{'-' * 60}")
|
||||
|
||||
try:
|
||||
# 数据预处理
|
||||
X_processed = self.preprocess_data(X_raw, preprocess_method)
|
||||
|
||||
# 训练模型
|
||||
result = self.train_single_model(X_processed, y, model_name,
|
||||
cv_folds, scoring, test_size, random_state, split_method)
|
||||
# 不再外部 Preprocessing——改传给 train_single_model 由 Pipeline 处理
|
||||
result = self.train_single_model(
|
||||
X_raw, y, model_name,
|
||||
cv_folds, scoring, test_size, random_state, split_method,
|
||||
preprocess_method=preprocess_method,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
# 保存模型
|
||||
# 保存模型（result['model'] 已是 sklearn Pipeline）
|
||||
metadata = {
|
||||
'target_column_name': target_column_name,
|
||||
'cv_mean': result['cv_mean'],
|
||||
'cv_std': result['cv_std'],
|
||||
'best_params': result['best_params'],
|
||||
'data_shape': X_processed.shape,
|
||||
'data_shape': X_raw.shape,
|
||||
'target_range': [float(y.min()), float(y.max())],
|
||||
'train_r2': result['train_r2'],
|
||||
'train_rmse': result['train_rmse'],
|
||||
@ -738,10 +760,13 @@ class WaterQualityModelingBatch:
|
||||
'test_mae': result['test_mae'],
|
||||
'train_size': result['train_size'],
|
||||
'test_size': result['test_size'],
|
||||
'split_method': result['split_method']
|
||||
'split_method': result['split_method'],
|
||||
# Pipeline 标记（便于旧 inference 路径兼容/诊断）
|
||||
'preprocess_method': preprocess_method,
|
||||
'is_pipeline': result.get('is_pipeline', False),
|
||||
}
|
||||
|
||||
self.save_model(result['model'], target_column_name,
|
||||
self.save_model(result['model'], target_column_name,
|
||||
f"{split_method}_{preprocess_method}",
|
||||
model_name, metadata)
|
||||
|
||||
|
||||
@ -38,6 +38,10 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
# sklearn Pipeline + 预处理 Transformer（避免 AutoML 训练时数据泄露）
|
||||
from sklearn.pipeline import Pipeline
|
||||
from src.preprocessing.spectral_Preprocessing import get_preprocessing_transformer
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 常量
|
||||
@ -189,8 +193,14 @@ def _get_search_space(model_name: str, trial) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def _make_objective(model_name: str, X: np.ndarray, y: np.ndarray,
|
||||
cv_folds: int, random_state: int):
|
||||
"""构造 Optuna objective（5 折 CV R²）。"""
|
||||
cv_folds: int, random_state: int,
|
||||
preproc_transformer=None):
|
||||
"""构造 Optuna objective（5 折 CV R²）。
|
||||
|
||||
Pipeline 化：把 preproc_transformer 与 builder 装成 Pipeline，
|
||||
cross_val_score 会 clone Pipeline 后每 fold 重 fit_transform，
|
||||
从而避免「scaler 在 full X 上 fit → split → CV」造成的数据泄露。
|
||||
"""
|
||||
from sklearn.model_selection import KFold, cross_val_score
|
||||
|
||||
def objective(trial):
|
||||
@ -199,9 +209,14 @@ def _make_objective(model_name: str, X: np.ndarray, y: np.ndarray,
|
||||
builder = _build_model(model_name, random_state=random_state)
|
||||
if builder is None:
|
||||
return -1.0
|
||||
model = builder(**params)
|
||||
base = builder(**params)
|
||||
pipe = Pipeline([
|
||||
('preproc', preproc_transformer if preproc_transformer is not None
|
||||
else get_preprocessing_transformer('None')),
|
||||
('model', base),
|
||||
])
|
||||
kf = KFold(n_splits=cv_folds, shuffle=True, random_state=random_state)
|
||||
scores = cross_val_score(model, X, y, cv=kf, scoring="r2", n_jobs=1)
|
||||
scores = cross_val_score(pipe, X, y, cv=kf, scoring="r2", n_jobs=1)
|
||||
return float(np.mean(scores))
|
||||
except Exception:
|
||||
return -1.0
|
||||
@ -210,14 +225,20 @@ def _make_objective(model_name: str, X: np.ndarray, y: np.ndarray,
|
||||
|
||||
|
||||
def _refit_full(model_name: str, best_params: Dict[str, Any],
|
||||
X: np.ndarray, y: np.ndarray, random_state: int):
|
||||
"""用 best params 在**全量数据**上 refit。"""
|
||||
X: np.ndarray, y: np.ndarray, random_state: int,
|
||||
preproc_transformer=None):
|
||||
"""用 best params 在**全量数据**上 refit，保存为 Pipeline（含 scaler 等状态）。"""
|
||||
builder = _build_model(model_name, random_state=random_state)
|
||||
if builder is None:
|
||||
return None
|
||||
model = builder(**best_params)
|
||||
model.fit(X, y)
|
||||
return model
|
||||
base = builder(**best_params)
|
||||
pipe = Pipeline([
|
||||
('preproc', preproc_transformer if preproc_transformer is not None
|
||||
else get_preprocessing_transformer('None')),
|
||||
('model', base),
|
||||
])
|
||||
pipe.fit(X, y)
|
||||
return pipe
|
||||
|
||||
|
||||
# ============================================================
|
||||
@ -360,18 +381,10 @@ def train_with_automl(
|
||||
feat_cols = [c for c in df.columns if c not in y_cols]
|
||||
X_all = df[feat_cols].values.astype(np.float64)
|
||||
|
||||
# ---- 3) 预处理（仅第一项） ----
|
||||
if preproc != "None":
|
||||
try:
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing
|
||||
processed = Preprocessing(preproc, df[feat_cols])
|
||||
if isinstance(processed, pd.DataFrame):
|
||||
X_all = processed.values.astype(np.float64)
|
||||
else:
|
||||
X_all = np.asarray(processed, dtype=np.float64)
|
||||
except Exception as e:
|
||||
notify("warning", f"预处理 {preproc} 失败: {e!r}，改用 None")
|
||||
preproc = "None"
|
||||
# ---- 3) 预处理（Pipeline 化：不再手工 Preprocessing，把 transformer 透传给 _make_objective/_refit_full） ----
|
||||
# 用一个 sklearn 兼容的 transformer 实例，后续每 trial 会被 clone 进 Pipeline.fit，
|
||||
# 保证 scaler 只在 train fold 上 fit（无数据泄露）。
|
||||
preproc_transformer = get_preprocessing_transformer(preproc)
|
||||
|
||||
# ---- 4) 检查 Optuna 是否可用 ----
|
||||
try:
|
||||
@ -427,7 +440,7 @@ def train_with_automl(
|
||||
sampler=optuna.samplers.TPESampler(seed=random_state),
|
||||
)
|
||||
study.optimize(
|
||||
_make_objective(model_name, X_sub, y_sub, cv_folds, random_state),
|
||||
_make_objective(model_name, X_sub, y_sub, cv_folds, random_state, preproc_transformer),
|
||||
n_trials=n_trials,
|
||||
timeout=per_model_timeout,
|
||||
show_progress_bar=False,
|
||||
@ -437,8 +450,8 @@ def train_with_automl(
|
||||
notify("warning", f"{tgt}/{model_name}: 全部 trial 失败（CV 全部 <= -1）")
|
||||
continue
|
||||
|
||||
# refit on FULL
|
||||
final_model = _refit_full(model_name, study.best_params, X_t, y_t, random_state)
|
||||
# refit on FULL（Pipeline 化：scaler 用全量 X_t 拟合一次）
|
||||
final_model = _refit_full(model_name, study.best_params, X_t, y_t, random_state, preproc_transformer)
|
||||
if final_model is None:
|
||||
continue
|
||||
|
||||
@ -453,6 +466,7 @@ def train_with_automl(
|
||||
"model_name": model_name,
|
||||
"metadata": {
|
||||
"automl": True,
|
||||
"is_pipeline": isinstance(final_model, Pipeline),
|
||||
"best_params": study.best_params,
|
||||
"cv_score": float(study.best_value),
|
||||
"n_trials_done": len(study.trials),
|
||||
|
||||
@ -12,7 +12,7 @@ warnings.filterwarnings('ignore')
|
||||
import sys
|
||||
import os
|
||||
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
|
||||
from src.core.utils.split_methods import spxy, ks
|
||||
|
||||
# try:
|
||||
@ -22,6 +22,7 @@ from src.core.utils.split_methods import spxy, ks
|
||||
|
||||
# 机器学习相关导入
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.pipeline import Pipeline
|
||||
|
||||
|
||||
class WaterQualityInference:
|
||||
@ -423,33 +424,20 @@ class WaterQualityInference:
|
||||
from src.utils.water_index import WaterQualityIndexCalculator
|
||||
calc = WaterQualityIndexCalculator()
|
||||
|
||||
# 提取纯计算方法（排除 find_closest_wavelength 和 calculate_all_indices，
|
||||
# 以及不返回 Series 的辅助方法）
|
||||
algorithm_methods = []
|
||||
for m in dir(calc):
|
||||
if m.startswith('_'):
|
||||
continue
|
||||
if m in ['find_closest_wavelength', 'calculate_all_indices']:
|
||||
continue
|
||||
attr = getattr(calc, m)
|
||||
if callable(attr):
|
||||
algorithm_methods.append(m)
|
||||
|
||||
original_col_count = spectra.shape[1]
|
||||
for algo_name in algorithm_methods:
|
||||
try:
|
||||
algo_func = getattr(calc, algo_name)
|
||||
result = algo_func(spectra)
|
||||
# 只追加返回 Series 且长度为样本数的合法结果
|
||||
if isinstance(result, pd.Series) and len(result) == len(spectra):
|
||||
spectra[algo_name] = result.values
|
||||
else:
|
||||
spectra[algo_name] = np.nan
|
||||
except Exception:
|
||||
spectra[algo_name] = np.nan
|
||||
|
||||
print(f"[特征补全] 完成！光谱列已扩充至 {spectra.shape[1]} 列"
|
||||
f"（追加了 {spectra.shape[1] - original_col_count} 个 WQI 指数）")
|
||||
# CSV 驱动的 WaterQualityIndexCalculator：所有公式名通过 list_available() 拿；
|
||||
# 一次性 calculate_many() 批量计算。彻底摆脱 dir(calc) 反射扫描 + 单 algo_func
|
||||
# 调用这种碎片化写法（Calculator 早已重构为公式驱动，不再有独立公式方法）。
|
||||
formulas = calc.list_available()
|
||||
if not formulas:
|
||||
print("[特征补全] Calculator 未持有任何公式，跳过补全")
|
||||
else:
|
||||
results_df = calc.calculate_many(formulas, spectra)
|
||||
# results_df 是列对齐的 WQI 计算结果（每列一个公式，行数=样本数）
|
||||
if isinstance(results_df, pd.DataFrame) and not results_df.empty:
|
||||
original_col_count = spectra.shape[1]
|
||||
spectra = pd.concat([spectra, results_df], axis=1)
|
||||
print(f"[特征补全] 完成！光谱列已扩充至 {spectra.shape[1]} 列"
|
||||
f"（追加了 {spectra.shape[1] - original_col_count} 个 WQI 指数）")
|
||||
except Exception as e:
|
||||
print(f"[特征补全] 失败，将使用原始光谱特征: {e}")
|
||||
|
||||
@ -471,6 +459,14 @@ class WaterQualityInference:
|
||||
|
||||
print(f"[特征对齐] 最终输入维度: {spectra.shape}")
|
||||
|
||||
# ---- Pipeline 化分支：模型内置 scaler/MSC.mean_spectrum_ 等状态时，跳过手动 Preprocessing ----
|
||||
if isinstance(model, Pipeline):
|
||||
print(f"[Pipeline] 检测到模型是 sklearn Pipeline，"
|
||||
f"其内置预处理步骤（{list(model.named_steps.keys())[0]}）将处理原始光谱，"
|
||||
f"无需外部 Preprocessing")
|
||||
return spectra.values
|
||||
|
||||
# ---- 兼容路径：旧 .joblib（裸模型 + preprocess_method 字符串）回退手动 Preprocessing ----
|
||||
try:
|
||||
# 应用预处理
|
||||
spectra_processed = Preprocessing(actual_preprocess_method, spectra)
|
||||
@ -479,7 +475,8 @@ class WaterQualityInference:
|
||||
if isinstance(spectra_processed, pd.DataFrame):
|
||||
spectra_processed = spectra_processed.values
|
||||
|
||||
print(f"预处理后数据形状: {spectra_processed.shape}")
|
||||
print(f" [Legacy] 旧裸模型 + 手动 Preprocessing({actual_preprocess_method}) 完成，"
|
||||
f"数据形状: {spectra_processed.shape}")
|
||||
|
||||
return spectra_processed
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
|
||||
from sklearn.model_selection import GridSearchCV, cross_val_score, KFold, train_test_split
|
||||
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
|
||||
from sklearn.cross_decomposition import PLSRegression
|
||||
from sklearn.pipeline import Pipeline
|
||||
from src.core.utils.split_methods import spxy, ks
|
||||
|
||||
# 第三方模型导入
|
||||
@ -46,7 +47,7 @@ CB_AVAILABLE = False # 注释掉catboost
|
||||
import sys
|
||||
import os
|
||||
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing
|
||||
from src.preprocessing.spectral_Preprocessing import Preprocessing, get_preprocessing_transformer
|
||||
|
||||
|
||||
class WaterQualityScatterBatch:
|
||||
@ -626,12 +627,18 @@ class WaterQualityScatterBatch:
|
||||
best_model_data = self.load_model(artifacts_path, model_file_prefix, model_name, folder_name)
|
||||
best_model = best_model_data['model']
|
||||
|
||||
# 应用相同的数据预处理
|
||||
X_processed = self.preprocess_data(X_raw, actual_preprocess_method)
|
||||
# 应用相同的数据预处理：Pipeline 模型自带 scaler，直接喂 raw；
|
||||
# 旧模型（裸模型）才需要手动 Preprocessing
|
||||
if isinstance(best_model, Pipeline):
|
||||
X_pred_input = X_raw
|
||||
print(f" [Pipeline] 模型自带预处理器，跳过外部 Preprocessing")
|
||||
else:
|
||||
X_pred_input = self.preprocess_data(X_raw, actual_preprocess_method)
|
||||
print(f" [Legacy] 旧裸模型，回退到手动 Preprocessing({actual_preprocess_method})")
|
||||
|
||||
# 使用相同的数据分割方法
|
||||
X_train, X_test, y_train, y_test = self.split_data(
|
||||
X_processed, y_true, method=split_method,
|
||||
X_pred_input, y_true, method=split_method,
|
||||
test_size=test_size, random_state=random_state
|
||||
)
|
||||
|
||||
|
||||
@ -236,6 +236,16 @@ class AISettingsDialog(QDialog):
|
||||
layout = QVBoxLayout(self)
|
||||
layout.setSpacing(12)
|
||||
|
||||
s = QSettings(AI_SETTINGS_ORG, AI_SETTINGS_APP)
|
||||
|
||||
# ★ 通用工具：把历史记录列表绑定到 QComboBox（去重追加）
|
||||
def _fill_combo_history(combo: QComboBox, history: list, preset: list):
|
||||
if not history:
|
||||
return
|
||||
for item in history:
|
||||
if item and item not in preset and combo.findText(item) == -1:
|
||||
combo.addItem(item)
|
||||
|
||||
# ── Provider ──────────────────────────────────────────────────────────
|
||||
provider_row = QHBoxLayout()
|
||||
provider_row.addWidget(QLabel("AI 引擎提供商:"))
|
||||
@ -243,7 +253,10 @@ class AISettingsDialog(QDialog):
|
||||
|
||||
# ★ 核心改动：开启可编辑模式，允许用户随意输入第三方代理商名字
|
||||
self._provider_combo.setEditable(True)
|
||||
self._provider_combo.addItems(["Aliyun", "Zhipu", "DeepSeek", "OpenAI", "Minimax", "Ollama"])
|
||||
provider_preset = ["Aliyun", "Zhipu", "DeepSeek", "OpenAI", "Minimax", "Ollama"]
|
||||
self._provider_combo.addItems(provider_preset)
|
||||
# 【历史倒灌】去重追加最近用过的 provider
|
||||
_fill_combo_history(self._provider_combo, s.value("ai_provider_history", [], type=list), provider_preset)
|
||||
self._provider_combo.setCurrentText(self._provider)
|
||||
|
||||
# 当文本改变时自动带出推荐配置
|
||||
@ -256,8 +269,12 @@ class AISettingsDialog(QDialog):
|
||||
# ── API Base URL ───────────────────────────────────────────────────────
|
||||
url_row = QHBoxLayout()
|
||||
url_row.addWidget(QLabel("API Base URL:"))
|
||||
self._url_edit = QLineEdit(self._api_base_url)
|
||||
self._url_edit.setPlaceholderText("填入兼容 OpenAI 规范的完整 URL")
|
||||
# 升级为可编辑下拉框：保留历史下拉 + 允许自由输入 111/localhost 等自定义值
|
||||
self._url_edit = QComboBox()
|
||||
self._url_edit.setEditable(True)
|
||||
self._url_edit.addItems(s.value("api_url_history", [], type=list))
|
||||
self._url_edit.setCurrentText(self._api_base_url)
|
||||
self._url_edit.setPlaceholderText("可输入 111/localhost 等自定义值并自动记忆")
|
||||
url_row.addWidget(self._url_edit, 1)
|
||||
layout.addLayout(url_row)
|
||||
|
||||
@ -273,11 +290,18 @@ class AISettingsDialog(QDialog):
|
||||
# ── 模型名称 ───────────────────────────────────────────────────────────
|
||||
model_row = QHBoxLayout()
|
||||
model_row.addWidget(QLabel("视觉模型:"))
|
||||
self._vision_edit = QLineEdit(self._vision_model)
|
||||
# 升级为可编辑下拉框：保留历史下拉 + 允许自由输入
|
||||
self._vision_edit = QComboBox()
|
||||
self._vision_edit.setEditable(True)
|
||||
self._vision_edit.addItems(s.value("vision_model_history", [], type=list))
|
||||
self._vision_edit.setCurrentText(self._vision_model)
|
||||
model_row.addWidget(self._vision_edit, 1)
|
||||
model_row.addSpacing(12)
|
||||
model_row.addWidget(QLabel("文本模型:"))
|
||||
self._text_edit = QLineEdit(self._text_model)
|
||||
self._text_edit = QComboBox()
|
||||
self._text_edit.setEditable(True)
|
||||
self._text_edit.addItems(s.value("text_model_history", [], type=list))
|
||||
self._text_edit.setCurrentText(self._text_model)
|
||||
model_row.addWidget(self._text_edit, 1)
|
||||
layout.addLayout(model_row)
|
||||
|
||||
@ -317,21 +341,41 @@ class AISettingsDialog(QDialog):
|
||||
provider_key = text.lower()
|
||||
if provider_key in AI_DEFAULTS:
|
||||
defaults = AI_DEFAULTS[provider_key]
|
||||
self._url_edit.setText(defaults["api_base_url"])
|
||||
self._vision_edit.setText(defaults["vision_model"])
|
||||
self._text_edit.setText(defaults["text_model"])
|
||||
self._url_edit.setCurrentText(defaults["api_base_url"])
|
||||
self._vision_edit.setCurrentText(defaults["vision_model"])
|
||||
self._text_edit.setCurrentText(defaults["text_model"])
|
||||
|
||||
def _save_and_close(self):
|
||||
"""持久化到 QSettings 并关闭。"""
|
||||
s = QSettings(AI_SETTINGS_ORG, AI_SETTINGS_APP)
|
||||
# 获取用户输入的文本（无论是选的还是自己打字的）
|
||||
provider = self._provider_combo.currentText().strip()
|
||||
url = self._url_edit.currentText().strip()
|
||||
vision = self._vision_edit.currentText().strip()
|
||||
text = self._text_edit.currentText().strip()
|
||||
|
||||
s.setValue("ai_provider", provider)
|
||||
s.setValue("api_base_url", self._url_edit.text().strip())
|
||||
s.setValue("api_base_url", url)
|
||||
s.setValue("api_key", self._key_edit.text().strip())
|
||||
s.setValue("vision_model", self._vision_edit.text().strip())
|
||||
s.setValue("text_model", self._text_edit.text().strip())
|
||||
s.setValue("vision_model", vision)
|
||||
s.setValue("text_model", text)
|
||||
s.setValue("timeout_s", self._timeout_spin.value())
|
||||
|
||||
# 【历史记录记忆拦截器】去重 + 最新在前 + 最多 10 条
|
||||
def _push_history(key: str, value: str):
|
||||
if not value:
|
||||
return
|
||||
hist = s.value(key, [], type=list) or []
|
||||
if value in hist:
|
||||
hist.remove(value)
|
||||
hist.insert(0, value)
|
||||
s.setValue(key, hist[:10])
|
||||
|
||||
_push_history("ai_provider_history", provider)
|
||||
_push_history("api_url_history", url)
|
||||
_push_history("vision_model_history", vision)
|
||||
_push_history("text_model_history", text)
|
||||
|
||||
s.sync()
|
||||
self.accept()
|
||||
|
||||
|
||||
@ -6,21 +6,46 @@ Step7 视图 - 水质光谱指数计算 (完美对齐卡片化重构版)
|
||||
|
||||
import os
|
||||
import sys
|
||||
import pandas as pd
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
from PyQt5.QtWidgets import (
|
||||
QVBoxLayout, QHBoxLayout, QGroupBox, QFormLayout,
|
||||
QLabel, QPushButton, QMessageBox, QListWidget,
|
||||
QListWidgetItem, QSizePolicy, QWidget
|
||||
QListWidgetItem, QSizePolicy, QWidget, QComboBox,
|
||||
)
|
||||
from PyQt5.QtCore import Qt
|
||||
from PyQt5.QtGui import QColor
|
||||
|
||||
from src.gui.core.event_bus import global_event_bus
|
||||
|
||||
from src.gui.components.custom_widgets import FileSelectWidget
|
||||
from src.gui.styles import ModernStylesheet
|
||||
|
||||
# ==========================================
|
||||
# 模块级字典 + 防穿透 ListWidget（与 Step10 对齐）
|
||||
# ==========================================
|
||||
CATEGORY_CHINESE_MAP = {
|
||||
'Total_Suspended_Matter': '总悬浮物 (TSM)',
|
||||
'Phycocyanin (BGA_PC)': '藻蓝蛋白 (PC)',
|
||||
'Turbidity': '浊度 (Turbidity)',
|
||||
'chlorophyll_a': '叶绿素a (Chl-a)',
|
||||
'Colored_Dissolved_Organic_Matter': '有色可溶性有机物 (CDOM)',
|
||||
'Secchi_Disk_Depth': '透明度 (SDD)',
|
||||
'Total_Nitrogen': '总氮 (TN)',
|
||||
'Total_Phosphorus': '总磷 (TP)',
|
||||
'Chemical_Oxygen_Demand': '化学需氧量 (COD)',
|
||||
'Ammonia_Nitrogen': '氨氮 (NH3-N)',
|
||||
'Dissolved_Oxygen': '溶解氧 (DO)'
|
||||
}
|
||||
|
||||
|
||||
class NoScrollPassListWidget(QListWidget):
|
||||
"""一个绝对不会把滚轮事件传给外层父组件的列表控件（与 Step10 同源）。"""
|
||||
def wheelEvent(self, event):
|
||||
super().wheelEvent(event)
|
||||
event.accept()
|
||||
|
||||
|
||||
class Step7InversionPanel(QWidget):
|
||||
"""步骤7：水质光谱指数计算"""
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import numpy as np
|
||||
from scipy import signal
|
||||
from sklearn.base import BaseEstimator, TransformerMixin
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.preprocessing import MinMaxScaler, StandardScaler
|
||||
import pandas as pd
|
||||
@ -174,3 +175,223 @@ def Preprocessing(method, input_spectrum, save_path=None):
|
||||
print("No such method of preprocessing!")
|
||||
output_spectrum = input_spectrum.values
|
||||
return output_spectrum
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# sklearn Pipeline 兼容的 Transformer 包装
|
||||
# ----------------------------------------------------------------------------
|
||||
# 设计目的：让 12 种预处理方法都能塞进 sklearn.pipeline.Pipeline，从而：
|
||||
# 1) 训练时 scaler/MSC mean spectrum 等状态被绑定在 Pipeline 内，
|
||||
# 避免传统"手动 Preprocessing(X_raw) → 拆分 → CV"的数据泄露链；
|
||||
# 2) 推理时直接 pipeline.predict(X_raw)，无需重新应用预处理；
|
||||
# 3) .joblib 内 model 字段即为完整 Pipeline，跨进程状态自包含。
|
||||
#
|
||||
# 注意：MMSTransformer/SSTransformer 直接复用 sklearn 自带的 MinMaxScaler/StandardScaler，
|
||||
# 不再封装（避免维护重复代码）；其余 9 种自定义方法各自写一个 TransformerMixin 子类。
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class _ArrayAsFloat64:
|
||||
"""统一的 ndarray 入口辅助（DataFrame/np.ndarray 都吃，输出 ndarray float64）"""
|
||||
@staticmethod
|
||||
def _to_ndarray(X):
|
||||
if isinstance(X, pd.DataFrame):
|
||||
return X.values.astype(np.float64)
|
||||
return np.asarray(X, dtype=np.float64)
|
||||
|
||||
|
||||
class IdentityTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""无预处理（None）—— 数据原样透传，shape 不变。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
return self._to_ndarray(X)
|
||||
|
||||
|
||||
class CTTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""均值中心化（CT）：每行减自身均值。shape 不变。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
return X - X.mean(axis=1, keepdims=True)
|
||||
|
||||
|
||||
class SNVTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""标准正态变换（SNV）：每行 (x - mean) / std。shape 不变。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
row_mean = X.mean(axis=1, keepdims=True)
|
||||
row_std = X.std(axis=1, keepdims=True)
|
||||
row_std = np.where(row_std == 0, 1.0, row_std)
|
||||
return (X - row_mean) / row_std
|
||||
|
||||
|
||||
class MATransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""移动平均平滑（MA）：每行卷积 np.ones(WSZ)/WSZ。shape 不变。"""
|
||||
def __init__(self, wsz: int = 11):
|
||||
self.wsz = wsz
|
||||
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
out = np.empty_like(X)
|
||||
WSZ = self.wsz
|
||||
r = np.arange(1, WSZ - 1, 2)
|
||||
for i in range(X.shape[0]):
|
||||
row = X[i]
|
||||
out0 = np.convolve(row, np.ones(WSZ, dtype=int), 'valid') / WSZ
|
||||
start = np.cumsum(row[:WSZ - 1])[::2] / r
|
||||
stop = (np.cumsum(row[:-WSZ:-1])[::2] / r)[::-1]
|
||||
out[i] = np.concatenate((start, out0, stop))
|
||||
return out
|
||||
|
||||
|
||||
class SGTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""Savitzky-Golay 平滑（SG）：每行调用 signal.savgol_filter。shape 不变。"""
|
||||
def __init__(self, w: int = 15, p: int = 2):
|
||||
self.w = w
|
||||
self.p = p
|
||||
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
return signal.savgol_filter(X, self.w, self.p, axis=1)
|
||||
|
||||
|
||||
class MSCTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""多元散射校正（MSC）：fit 阶段计算训练集平均光谱；transform 阶段对每行
|
||||
以平均光谱为参考做线性回归 (k, b)，输出 (x - b) / k。shape 不变。
|
||||
|
||||
注意：fit 阶段对每行分别拟合一次回归取 (k, b) 仅用于兼容旧实现，标准 MSC
|
||||
只存储 mean_spectrum_。这里为了与原代码行为一致，保留 per-row 拟合路径。
|
||||
"""
|
||||
def fit(self, X, y=None):
|
||||
X = self._to_ndarray(X)
|
||||
self.mean_spectrum_ = X.mean(axis=0)
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
mean = self.mean_spectrum_
|
||||
out = np.empty_like(X)
|
||||
lr = LinearRegression()
|
||||
for i in range(X.shape[0]):
|
||||
y = X[i]
|
||||
lr.fit(mean.reshape(-1, 1), y.reshape(-1, 1))
|
||||
k = lr.coef_[0, 0]
|
||||
b = lr.intercept_[0]
|
||||
out[i] = (y - b) / (k if k != 0 else 1.0)
|
||||
return out
|
||||
|
||||
|
||||
class D1Transformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""一阶导数（D1）：每行 np.diff。shape 从 (n, p) → (n, p-1)。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
return np.diff(X, axis=1)
|
||||
|
||||
|
||||
class D2Transformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""二阶导数（D2）：每行二次 np.diff。shape 从 (n, p) → (n, p-2)。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
return np.diff(X, n=2, axis=1)
|
||||
|
||||
|
||||
class DTTransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""趋势校正（DT）：每行对自身索引做线性回归，减去趋势线。shape 不变。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
n_cols = X.shape[1]
|
||||
x = np.asarray(range(n_cols), dtype=np.float32).reshape(-1, 1)
|
||||
out = np.empty_like(X)
|
||||
lr = LinearRegression()
|
||||
for i in range(X.shape[0]):
|
||||
row = X[i]
|
||||
lr.fit(x, row.reshape(-1, 1))
|
||||
trend = (x @ lr.coef_.T + lr.intercept_).ravel()
|
||||
out[i] = row - trend
|
||||
return out
|
||||
|
||||
|
||||
class WVAETransformer(TransformerMixin, BaseEstimator, _ArrayAsFloat64):
|
||||
"""小波变换（WVAE）：每行调用 pywt 阈值去噪重构。shape 可能略有变化。"""
|
||||
def fit(self, X, y=None):
|
||||
return self
|
||||
|
||||
def transform(self, X):
|
||||
X = self._to_ndarray(X)
|
||||
w = pywt.Wavelet('db8')
|
||||
maxlev = pywt.dwt_max_level(X.shape[1], w.dec_len)
|
||||
out = np.empty_like(X)
|
||||
for i in range(X.shape[0]):
|
||||
row = X[i]
|
||||
coeffs = pywt.wavedec(row, 'db8', level=maxlev)
|
||||
for ci in range(1, len(coeffs)):
|
||||
coeffs[ci] = pywt.threshold(coeffs[ci], 0.04 * max(np.abs(coeffs[ci])) if coeffs[ci].size else 1.0)
|
||||
reconstructed = pywt.waverec(coeffs, 'db8')
|
||||
# waverec 可能比原信号长 1 元素（边界效应），裁剪对齐
|
||||
out[i] = reconstructed[:X.shape[1]]
|
||||
return out
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# 工厂函数：根据方法名返回对应的 sklearn 兼容 Transformer（None 表示无预处理）
|
||||
# ============================================================================
|
||||
|
||||
_PREPROCESSING_TRANSFORMERS = {
|
||||
'None': IdentityTransformer,
|
||||
'MMS': MinMaxScaler, # sklearn 自带
|
||||
'SS': StandardScaler, # sklearn 自带
|
||||
'CT': CTTransformer,
|
||||
'SNV': SNVTransformer,
|
||||
'MA': MATransformer,
|
||||
'SG': SGTransformer,
|
||||
'MSC': MSCTransformer,
|
||||
'D1': D1Transformer,
|
||||
'D2': D2Transformer,
|
||||
'DT': DTTransformer,
|
||||
'WVAE': WVAETransformer,
|
||||
}
|
||||
|
||||
|
||||
def get_preprocessing_transformer(method: str):
|
||||
"""根据预处理方法名返回 sklearn 兼容的 Transformer 实例。
|
||||
|
||||
- method 为 "None" 或 None：返回 IdentityTransformer（等价于无处理）
|
||||
- method 为 "MMS"/"SS"：直接返回 sklearn 自带 MinMaxScaler/StandardScaler
|
||||
- method 为 "CT"/"SNV"/"MA"/"SG"/"MSC"/"D1"/"D2"/"DT"/"WVAE"：返回对应包装类
|
||||
- method 不识别：返回 IdentityTransformer + 打印警告（与原 Preprocessing 行为一致）
|
||||
|
||||
Args:
|
||||
method: 预处理方法名（大小写敏感，与 Preprocessing() 一致）
|
||||
|
||||
Returns:
|
||||
sklearn 兼容的 Transformer 实例（可直接放入 Pipeline）
|
||||
"""
|
||||
if method is None:
|
||||
return IdentityTransformer()
|
||||
if method not in _PREPROCESSING_TRANSFORMERS:
|
||||
print(f"未知预处理方法 '{method}'，回退为 IdentityTransformer")
|
||||
return IdentityTransformer()
|
||||
cls = _PREPROCESSING_TRANSFORMERS[method]
|
||||
return cls()
|
||||
|
||||
Reference in New Issue
Block a user