Initial commit of WQ_GUI

src/preprocessing/process_water_quality_data.py

@@ -0,0 +1,223 @@
+import threading  # 放在你的其他 import 之前
+if not hasattr(threading.Thread, "isAlive"):
+    threading.Thread.isAlive = threading.Thread.is_alive  # 给旧调试器一个别名
+
+import warnings
+import os
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+warnings.filterwarnings("ignore")
+
+
+def detect_outliers_iqr(data: pd.DataFrame, column: str) -> pd.Series:
+    """使用 IQR 方法检测异常值，返回与 data 同索引的布尔序列"""
+    s = pd.to_numeric(data[column], errors="coerce")
+    q1 = s.quantile(0.25)
+    q3 = s.quantile(0.75)
+    iqr = q3 - q1
+    lower = q1 - 1.5 * iqr
+    upper = q3 + 1.5 * iqr
+    mask = (s < lower) | (s > upper)
+    # 对 NaN 不判为异常
+    mask = mask.fillna(False)
+    mask.index = data.index
+    return mask
+
+
+def detect_outliers_zscore(data: pd.DataFrame, column: str, threshold: float = 3.0) -> pd.Series:
+    """使用 Z-score 方法检测异常值，返回与 data 同索引的布尔序列"""
+    s = pd.to_numeric(data[column], errors="coerce")
+    z = pd.Series(stats.zscore(s.dropna()), index=s.dropna().index)
+    mask = (z.abs() > threshold).reindex(data.index).fillna(False)
+    return mask
+
+
+def read_csv_robust(path, **kwargs):
+    """
+    尝试多种编码读取 CSV；成功即返回 DataFrame。
+    kwargs 会透传给 pd.read_csv（比如 sep、dtype 等）。
+    """
+    # 按出现概率排序；优先无损 + 常见中文编码
+    encodings = [
+        "utf-8", "utf-8-sig",
+        "gbk", "gb18030", "cp936",
+        "utf-16", "utf-16le", "utf-16be",
+        "cp1252", "big5",
+        "latin1",  # 最后兜底（能读但中文会变乱码）
+    ]
+    errors_modes = ["strict", "replace"]  # 先严格，失败再替换非法字符
+
+    last_err = None
+    for enc in encodings:
+        for emode in errors_modes:
+            try:
+                return pd.read_csv(path, encoding=enc, **kwargs)
+            except Exception as e:
+                last_err = e
+                continue
+    # 如果全失败，抛出最后一个错误
+    raise last_err
+
+
+def _decimal_len(v) -> float:
+    """计算数值或字符串小数点后的位数；若无法计算返回 NaN"""
+    if pd.isna(v):
+        return np.nan
+    try:
+        # 统一成字符串处理
+        s = str(v)
+        if "." not in s:
+            return 0
+        frac = s.split(".", 1)[1]
+        # 去掉科学计数法中的尾随部分（如 '1.234e-05'）
+        frac = frac.split("e")[0].split("E")[0]
+        return len(frac)
+    except Exception:
+        return np.nan
+
+
+def process_water_quality_data(input_file: str, output_file: str):
+    """
+    处理水质数据 CSV 文件
+
+    参数:
+    input_file: 输入 CSV 文件路径
+    output_file: 输出 CSV 文件路径
+    """
+    # 0) 读取
+    print("正在读取 CSV 文件...")
+    df = read_csv_robust(input_file)
+    print(f"原始数据形状: {df.shape}")
+    print(f"列名: {list(df.columns)}")
+
+    # 1) 经纬度精度筛选（小数位 >= 7）
+    print("\n正在筛选经纬度精度(小数位>=7)...")
+    initial_count = len(df)
+
+    for col in ["经度", "纬度"]:
+        if col in df.columns:
+            dec_len = df[col].apply(_decimal_len)
+            keep_mask = dec_len >= 7
+            dropped = (~keep_mask).sum()
+            df = df[keep_mask].copy()
+            print(f"列 {col}: 去除了 {int(dropped)} 行（保留 {len(df)} 行）")
+
+    after_coord_filter = len(df)
+    print(f"经纬度精度筛选后剩余: {after_coord_filter} 行 (去除了 {initial_count - after_coord_filter} 行)")
+
+    # 2) 异常值检测（IQR）- 只删除异常值，不删除整行
+    print("\n正在检测异常值(IQR)...")
+    # 数值列
+    numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()
+
+    # 排除不检测的列
+    exclude_columns = ["时间", "测量点", "纬度", "经度"]
+    if "原始" in df.columns:
+        exclude_columns.append("原始")
+
+    columns_to_check = [c for c in numeric_columns if c not in exclude_columns]
+    print(f"将检测以下列的异常值: {columns_to_check}")
+
+    df_clean = df.copy()
+    total_outliers_removed = 0
+    
+    for column in columns_to_check:
+        if column in df_clean.columns and df_clean[column].notna().sum() > 0:
+            col_mask = detect_outliers_iqr(df_clean, column)
+            outlier_count = int(col_mask.sum())
+            print(f'列 "{column}" 检测到 {outlier_count} 个异常值，将其设为 NaN')
+            # 只将异常值设为 NaN，不删除整行
+            df_clean.loc[col_mask, column] = np.nan
+            total_outliers_removed += outlier_count
+
+    after_outlier_filter = len(df_clean)
+    print(f"异常值处理完成: 保留 {after_outlier_filter} 行数据，共处理了 {total_outliers_removed} 个异常值")
+
+    # 3) 去除 "原始" 列（若存在）
+    if "原始" in df_clean.columns:
+        df_clean = df_clean.drop(columns=["原始"])
+        print('已去除 "原始" 列')
+
+    # 4) 字段类型处理：尽量把“时间”转为 datetime
+    if "时间" in df_clean.columns:
+        try:
+            df_clean["时间"] = pd.to_datetime(df_clean["时间"], errors="coerce")
+        except Exception:
+            pass
+
+    # 5) 按测量点统计平均值
+    print("\n正在按测量点统计平均值...")
+    if "测量点" not in df_clean.columns:
+        print('错误：未找到 "测量点" 列')
+        return
+
+    # 构建聚合字典
+    agg_dict = {}
+    if "时间" in df_clean.columns and np.issubdtype(df_clean["时间"].dtype, np.datetime64):
+        # 时间取平均（等价于时间戳平均）
+        agg_dict["时间"] = "mean"
+    elif "时间" in df_clean.columns:
+        # 如果不是时间类型，保留最常见值以避免无意义的字符串平均
+        agg_dict["时间"] = lambda s: s.mode().iloc[0] if not s.mode().empty else s.dropna().iloc[0] if s.dropna().size else np.nan
+
+    for col in ["纬度", "经度"]:
+        if col in df_clean.columns:
+            agg_dict[col] = "mean"
+
+    # 其余数值列取均值
+    for col in df_clean.select_dtypes(include=[np.number]).columns:
+        if col not in ["纬度", "经度"]:
+            agg_dict[col] = "mean"
+
+    grouped = df_clean.groupby("测量点", as_index=False).agg(agg_dict)
+
+    print(f"统计完成，共 {len(grouped)} 个测量点")
+    print(f"输出数据形状: {grouped.shape}")
+
+    # 6) 去除"时间"和"测量点"列
+    columns_to_drop = []
+    if "时间" in grouped.columns:
+        columns_to_drop.append("时间")
+    if "测量点" in grouped.columns:
+        columns_to_drop.append("测量点")
+    
+    if columns_to_drop:
+        grouped = grouped.drop(columns=columns_to_drop)
+        print(f"已去除列: {columns_to_drop}")
+        print(f"去除列后数据形状: {grouped.shape}")
+
+    # 7) 保存
+    os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+    grouped.to_csv(output_file, index=False, encoding="utf-8-sig")
+    print(f"\n处理完成！结果已保存到: {output_file}")
+
+    # 摘要
+    print("\n=== 处理结果摘要 ===")
+    print(f"原始数据行数: {initial_count}")
+    print(f"经纬度精度筛选后: {after_coord_filter}")
+    print(f"异常值筛选后: {after_outlier_filter}")
+    print(f"最终统计结果: {len(grouped)} 个测量点")
+
+    return grouped
+
+
+def main():
+    """主函数"""
+    input_file = r"D:\BaiduNetdiskDownload\yaobao\csv\input.csv"
+    output_file =r"D:\BaiduNetdiskDownload\yaobao\csv\output_test.csv"
+
+    if not output_file:
+        output_file = "processed_water_quality.csv"
+
+    try:
+        _ = process_water_quality_data(input_file, output_file)
+    except FileNotFoundError as e:
+        print(f"文件未找到：{e}")
+    except Exception as e:
+        print(f"处理失败：{e}")
+
+
+if __name__ == "__main__":
+    main()