duxingchen/WQ_GUI
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit of WQ_GUI

Browse Source
This commit is contained in:
zhanghuilai
2026-04-08 15:25:08 +08:00
commit 91e36407ae
302 changed files with 40872 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/preprocessing/__init__.py Normal file
View File

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

223
src/preprocessing/process_water_quality_data.py Normal file
View File

@ -0,0 +1,223 @@
import threading # 放在你的其他 import 之前
if not hasattr(threading.Thread, "isAlive"):
threading.Thread.isAlive = threading.Thread.is_alive # 给旧调试器一个别名
import warnings
import os
import numpy as np
import pandas as pd
from scipy import stats
warnings.filterwarnings("ignore")
def detect_outliers_iqr(data: pd.DataFrame, column: str) -> pd.Series:
"""使用 IQR 方法检测异常值，返回与 data 同索引的布尔序列"""
s = pd.to_numeric(data[column], errors="coerce")
q1 = s.quantile(0.25)
q3 = s.quantile(0.75)
iqr = q3 - q1
lower = q1 - 1.5 * iqr
upper = q3 + 1.5 * iqr
mask = (s < lower) | (s > upper)
# 对 NaN 不判为异常
mask = mask.fillna(False)
mask.index = data.index
return mask
def detect_outliers_zscore(data: pd.DataFrame, column: str, threshold: float = 3.0) -> pd.Series:
"""使用 Z-score 方法检测异常值，返回与 data 同索引的布尔序列"""
s = pd.to_numeric(data[column], errors="coerce")
z = pd.Series(stats.zscore(s.dropna()), index=s.dropna().index)
mask = (z.abs() > threshold).reindex(data.index).fillna(False)
return mask
def read_csv_robust(path, **kwargs):
"""
尝试多种编码读取 CSV；成功即返回 DataFrame。
kwargs 会透传给 pd.read_csv（比如 sep、dtype 等）。
"""
# 按出现概率排序；优先无损 + 常见中文编码
encodings = [
"utf-8", "utf-8-sig",
"gbk", "gb18030", "cp936",
"utf-16", "utf-16le", "utf-16be",
"cp1252", "big5",
"latin1", # 最后兜底（能读但中文会变乱码）
]
errors_modes = ["strict", "replace"] # 先严格，失败再替换非法字符
last_err = None
for enc in encodings:
for emode in errors_modes:
try:
return pd.read_csv(path, encoding=enc, **kwargs)
except Exception as e:
last_err = e
continue
# 如果全失败，抛出最后一个错误
raise last_err
def _decimal_len(v) -> float:
"""计算数值或字符串小数点后的位数；若无法计算返回 NaN"""
if pd.isna(v):
return np.nan
try:
# 统一成字符串处理
s = str(v)
if "." not in s:
return 0
frac = s.split(".", 1)[1]
# 去掉科学计数法中的尾随部分（如 '1.234e-05'）
frac = frac.split("e")[0].split("E")[0]
return len(frac)
except Exception:
return np.nan
def process_water_quality_data(input_file: str, output_file: str):
"""
处理水质数据 CSV 文件
参数:
input_file: 输入 CSV 文件路径
output_file: 输出 CSV 文件路径
"""
# 0) 读取
print("正在读取 CSV 文件...")
df = read_csv_robust(input_file)
print(f"原始数据形状: {df.shape}")
print(f"列名: {list(df.columns)}")
# 1) 经纬度精度筛选（小数位 >= 7）
print("\n正在筛选经纬度精度(小数位>=7)...")
initial_count = len(df)
for col in ["经度", "纬度"]:
if col in df.columns:
dec_len = df[col].apply(_decimal_len)
keep_mask = dec_len >= 7
dropped = (~keep_mask).sum()
df = df[keep_mask].copy()
print(f"列 {col}: 去除了 {int(dropped)} 行（保留 {len(df)} 行）")
after_coord_filter = len(df)
print(f"经纬度精度筛选后剩余: {after_coord_filter} 行 (去除了 {initial_count - after_coord_filter} 行)")
# 2) 异常值检测（IQR）- 只删除异常值，不删除整行
print("\n正在检测异常值(IQR)...")
# 数值列
numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
# 排除不检测的列
exclude_columns = ["时间", "测量点", "纬度", "经度"]
if "原始" in df.columns:
exclude_columns.append("原始")
columns_to_check = [c for c in numeric_columns if c not in exclude_columns]
print(f"将检测以下列的异常值: {columns_to_check}")
df_clean = df.copy()
total_outliers_removed = 0
for column in columns_to_check:
if column in df_clean.columns and df_clean[column].notna().sum() > 0:
col_mask = detect_outliers_iqr(df_clean, column)
outlier_count = int(col_mask.sum())
print(f'列 "{column}" 检测到 {outlier_count} 个异常值，将其设为 NaN')
# 只将异常值设为 NaN，不删除整行
df_clean.loc[col_mask, column] = np.nan
total_outliers_removed += outlier_count
after_outlier_filter = len(df_clean)
print(f"异常值处理完成: 保留 {after_outlier_filter} 行数据，共处理了 {total_outliers_removed} 个异常值")
# 3) 去除 "原始" 列（若存在）
if "原始" in df_clean.columns:
df_clean = df_clean.drop(columns=["原始"])
print('已去除 "原始" 列')
# 4) 字段类型处理：尽量把“时间”转为 datetime
if "时间" in df_clean.columns:
try:
df_clean["时间"] = pd.to_datetime(df_clean["时间"], errors="coerce")
except Exception:
pass
# 5) 按测量点统计平均值
print("\n正在按测量点统计平均值...")
if "测量点" not in df_clean.columns:
print('错误：未找到 "测量点" 列')
return
# 构建聚合字典
agg_dict = {}
if "时间" in df_clean.columns and np.issubdtype(df_clean["时间"].dtype, np.datetime64):
# 时间取平均（等价于时间戳平均）
agg_dict["时间"] = "mean"
elif "时间" in df_clean.columns:
# 如果不是时间类型，保留最常见值以避免无意义的字符串平均
agg_dict["时间"] = lambda s: s.mode().iloc[0] if not s.mode().empty else s.dropna().iloc[0] if s.dropna().size else np.nan
for col in ["纬度", "经度"]:
if col in df_clean.columns:
agg_dict[col] = "mean"
# 其余数值列取均值
for col in df_clean.select_dtypes(include=[np.number]).columns:
if col not in ["纬度", "经度"]:
agg_dict[col] = "mean"
grouped = df_clean.groupby("测量点", as_index=False).agg(agg_dict)
print(f"统计完成，共 {len(grouped)} 个测量点")
print(f"输出数据形状: {grouped.shape}")
# 6) 去除"时间"和"测量点"列
columns_to_drop = []
if "时间" in grouped.columns:
columns_to_drop.append("时间")
if "测量点" in grouped.columns:
columns_to_drop.append("测量点")
if columns_to_drop:
grouped = grouped.drop(columns=columns_to_drop)
print(f"已去除列: {columns_to_drop}")
print(f"去除列后数据形状: {grouped.shape}")
# 7) 保存
os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
grouped.to_csv(output_file, index=False, encoding="utf-8-sig")
print(f"\n处理完成！结果已保存到: {output_file}")
# 摘要
print("\n=== 处理结果摘要 ===")
print(f"原始数据行数: {initial_count}")
print(f"经纬度精度筛选后: {after_coord_filter}")
print(f"异常值筛选后: {after_outlier_filter}")
print(f"最终统计结果: {len(grouped)} 个测量点")
return grouped
def main():
"""主函数"""
input_file = r"D:\BaiduNetdiskDownload\yaobao\csv\input.csv"
output_file =r"D:\BaiduNetdiskDownload\yaobao\csv\output_test.csv"
if not output_file:
output_file = "processed_water_quality.csv"
try:
_ = process_water_quality_data(input_file, output_file)
except FileNotFoundError as e:
print(f"文件未找到：{e}")
except Exception as e:
print(f"处理失败：{e}")
if __name__ == "__main__":
main()

157
src/preprocessing/spectral_Preprocessing.py Normal file
View File

@ -0,0 +1,157 @@
import numpy as np
from scipy import signal
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import pandas as pd
import pywt
from copy import deepcopy
import joblib # 用于保存和加载模型
# 最大最小值归一化
def MMS(input_spectrum):
output_spectrum = MinMaxScaler().fit_transform(input_spectrum)
return output_spectrum
# 标准化
def SS(input_spectrum, save_path=None):
# 初始化 StandardScaler 并拟合数据
scaler = StandardScaler()
output_spectrum = scaler.fit_transform(input_spectrum)
# 如果指定了保存路径，保存 scaler 对象
if save_path:
joblib.dump(scaler, save_path)
print(f"Scaler parameters saved to {save_path}")
return output_spectrum
# 均值中心化
def CT(input_spectrum):
output_spectrum = deepcopy(input_spectrum)
for i in range(output_spectrum.shape[0]):
MEAN = np.mean(output_spectrum[i])
output_spectrum[i] = output_spectrum[i] - MEAN
return output_spectrum
# 标准正态变换
def SNV(input_spectrum):
if not isinstance(input_spectrum, pd.DataFrame):
raise ValueError("Input spectrum must be a Pandas DataFrame")
data_average = input_spectrum.mean(axis=1)
data_std = input_spectrum.std(axis=1)
data_std = data_std.replace(0, 1)
output_spectrum = (input_spectrum.sub(data_average, axis=0)).div(data_std, axis=0)
return output_spectrum
# 移动平均平滑
def MA(input_spectrum, WSZ=11):
output_spectrum = deepcopy(input_spectrum)
for i in range(output_spectrum.shape[0]):
out0 = np.convolve(output_spectrum[i], np.ones(WSZ, dtype=int), 'valid') / WSZ
r = np.arange(1, WSZ - 1, 2)
start = np.cumsum(output_spectrum[i, :WSZ - 1])[::2] / r
stop = (np.cumsum(output_spectrum[i, :-WSZ:-1])[::2] / r)[::-1]
output_spectrum[i] = np.concatenate((start, out0, stop))
return output_spectrum
# Savitzky-Golay平滑滤波
def SG(input_spectrum, w=15, p=2):
output_spectrum = signal.savgol_filter(input_spectrum, w, p)
return output_spectrum
# 一阶导数
def D1(input_spectrum):
n, p = input_spectrum.shape
output_spectrum = np.ones((n, p - 1))
for i in range(n):
output_spectrum[i] = np.diff(input_spectrum[i])
return output_spectrum
# 二阶导数
def D2(input_spectrum):
temp2 = (pd.DataFrame(input_spectrum)).diff(axis=1)
temp3 = np.delete(temp2.values, 0, axis=1)
temp4 = (pd.DataFrame(temp3)).diff(axis=1)
output_spectrum = np.delete(temp4.values, 0, axis=1)
return output_spectrum
# 趋势校正
def DT(input_spectrum):
lenth = input_spectrum.shape[1]
x = np.asarray(range(lenth), dtype=np.float32)
output_spectrum = np.array(input_spectrum)
l = LinearRegression()
for i in range(output_spectrum.shape[0]):
l.fit(x.reshape(-1, 1), output_spectrum[i].reshape(-1, 1))
k = l.coef_
b = l.intercept_
for j in range(output_spectrum.shape[1]):
output_spectrum[i][j] = output_spectrum[i][j] - (j * k + b)
return output_spectrum
# 多元散射校正
def MSC(input_spectrum):
n, p = input_spectrum.shape
output_spectrum = np.ones((n, p))
mean = np.mean(input_spectrum, axis=0)
for i in range(n):
y = input_spectrum[i, :]
l = LinearRegression()
l.fit(mean.reshape(-1, 1), y.reshape(-1, 1))
k = l.coef_
b = l.intercept_
output_spectrum[i, :] = (y - b) / k
return output_spectrum
# 小波变换
def wave(input_spectrum):
def wave_(input_spectrum_row):
w = pywt.Wavelet('db8')
maxlev = pywt.dwt_max_level(len(input_spectrum_row), w.dec_len)
coeffs = pywt.wavedec(input_spectrum_row, 'db8', level=maxlev)
threshold = 0.04
for i in range(1, len(coeffs)):
coeffs[i] = pywt.threshold(coeffs[i], threshold * max(coeffs[i]))
output_spectrum_row = pywt.waverec(coeffs, 'db8')
return output_spectrum_row
output_spectrum = None
for i in range(input_spectrum.shape[0]):
if i == 0:
output_spectrum = wave_(input_spectrum[i])
else:
output_spectrum = np.vstack((output_spectrum, wave_(input_spectrum[i])))
return output_spectrum
# 通用预处理函数
def Preprocessing(method, input_spectrum):
if isinstance(input_spectrum, np.ndarray):
input_spectrum = pd.DataFrame(input_spectrum)
if method == "None":
output_spectrum = input_spectrum
elif method == 'MMS':
output_spectrum = MMS(input_spectrum.values)
elif method == 'SS':
output_spectrum = SS(input_spectrum.values, r'E:\code\WQ\models/scaler_params.pkl')
elif method == 'CT':
output_spectrum = CT(input_spectrum.values)
elif method == 'SNV':
output_spectrum = SNV(input_spectrum)
elif method == 'MA':
output_spectrum = MA(input_spectrum.values)
elif method == 'SG':
output_spectrum = SG(input_spectrum.values)
elif method == 'MSC':
output_spectrum = MSC(input_spectrum.values)
elif method == 'D1':
output_spectrum = D1(input_spectrum.values)
elif method == 'D2':
output_spectrum = D2(input_spectrum.values)
elif method == 'DT':
output_spectrum = DT(input_spectrum.values)
elif method == 'WVAE':
output_spectrum = wave(input_spectrum.values)
else:
print("No such method of preprocessing!")
output_spectrum = input_spectrum.values
return output_spectrum