Initial commit: GasFlux project with core processing pipelines
This commit is contained in:
0
src/__init__.py
Normal file
0
src/__init__.py
Normal file
29
src/gasflux/__init__.py
Normal file
29
src/gasflux/__init__.py
Normal file
@ -0,0 +1,29 @@
|
||||
"""Init file for gasflux package."""
|
||||
|
||||
__version__ = "0.2.1" # managed by semantic versioning
|
||||
|
||||
from . import (
|
||||
background,
|
||||
cli,
|
||||
gas,
|
||||
interpolation,
|
||||
ml,
|
||||
plotting,
|
||||
pre_processing,
|
||||
processing,
|
||||
processing_pipelines,
|
||||
reporting,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"background",
|
||||
"cli",
|
||||
"gas",
|
||||
"interpolation",
|
||||
"ml",
|
||||
"plotting",
|
||||
"pre_processing",
|
||||
"processing",
|
||||
"processing_pipelines",
|
||||
"reporting",
|
||||
]
|
||||
64
src/gasflux/background.py
Normal file
64
src/gasflux/background.py
Normal file
@ -0,0 +1,64 @@
|
||||
"""Baselining functions."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pybaselines as pybs
|
||||
from . import plotting
|
||||
|
||||
# 自定义阈值函数，避免依赖scikit-image
|
||||
def custom_threshold(data):
|
||||
"""
|
||||
简单的三角阈值算法实现，避免依赖scikit-image
|
||||
使用直方图的三角法来确定阈值
|
||||
"""
|
||||
if len(data) == 0:
|
||||
return 0
|
||||
|
||||
# 计算直方图
|
||||
hist, bin_edges = np.histogram(data, bins=256)
|
||||
bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
|
||||
|
||||
# 找到最大峰值
|
||||
max_idx = np.argmax(hist)
|
||||
|
||||
# 如果最大峰在边缘，使用中位数作为阈值
|
||||
if max_idx == 0 or max_idx == len(hist) - 1:
|
||||
return np.median(data)
|
||||
|
||||
# 使用简单的阈值策略：最大峰右侧的第一个局部最小值
|
||||
# 这里简化为使用均值作为阈值
|
||||
return np.mean(data)
|
||||
|
||||
|
||||
def algorithmic_baseline(
|
||||
df: pd.DataFrame,
|
||||
gas: str,
|
||||
algorithmic_baseline_settings: dict,
|
||||
):
|
||||
df = df.copy()
|
||||
algorithm = algorithmic_baseline_settings["algorithm"]
|
||||
settings = algorithmic_baseline_settings.get(algorithm, {}).copy()
|
||||
if settings.get("threshold") == "custom":
|
||||
settings["threshold"] = custom_threshold
|
||||
if len(df) < 20:
|
||||
raise ValueError("Dataframe must contain at least 20 rows for background correction.")
|
||||
index = np.arange(len(df))
|
||||
baseline_fitter = pybs.Baseline(index, check_finite=False)
|
||||
fit = getattr(baseline_fitter, algorithm)
|
||||
bkg, params = fit(df[gas], **settings)
|
||||
bkg_points = params["mask"]
|
||||
df[f"{gas}_normalised"] = df[gas] - bkg
|
||||
df[f"{gas}_fit"] = bkg
|
||||
background = (df[gas] - bkg)[bkg_points]
|
||||
signal = (df[gas] - bkg)[~bkg_points]
|
||||
df[f"{gas}_signal"] = np.invert(bkg_points)
|
||||
fig = plotting.background_plotting(df, gas)
|
||||
output_text = (
|
||||
f"Baseline algorithm: {algorithm}\n"
|
||||
f"Positive and negative 95% percentile of baseline: {np.percentile(background, 2.5):.2f} ppm, \
|
||||
{np.percentile(background, 97.5):.2f} ppm\n"
|
||||
f"Mean of baseline: {np.mean(background):.2f} ppm\n"
|
||||
f"Minimum and maximum of baseline: {np.min(background):.2f} ppm, {np.max(background):.2f} ppm\n"
|
||||
f"Signal points: {len(signal)}; background points: {len(background)}\n"
|
||||
)
|
||||
return df, fig, output_text
|
||||
279
src/gasflux/cli.py
Normal file
279
src/gasflux/cli.py
Normal file
@ -0,0 +1,279 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
from colorama import init, Fore, Style
|
||||
from src.gasflux.processing_pipelines import process_main
|
||||
|
||||
init()
|
||||
|
||||
|
||||
def find_config_file(path: Path, recursive: bool = True):
|
||||
if recursive:
|
||||
config_files = [file for file in path.rglob("*.yaml") if "gasflux_config" in file.read_text()]
|
||||
else:
|
||||
config_files = [file for file in path.glob("*.yaml") if "gasflux_config" in file.read_text()]
|
||||
if len(config_files) == 1:
|
||||
return config_files[0]
|
||||
elif len(config_files) > 1:
|
||||
raise ValueError(
|
||||
"Multiple candidate config files found: {}".format(", ".join(str(file) for file in config_files))
|
||||
)
|
||||
else:
|
||||
raise FileNotFoundError("No config file found in the supplied path or its child folders")
|
||||
|
||||
|
||||
def process_command(data_path: str, config_path: str, test: bool):
|
||||
if test:
|
||||
data_file = Path(__file__).parent / "testdata" / "testdata.csv"
|
||||
config_file = Path(__file__).parent / "testdata" / "testconfig.yaml"
|
||||
process_main(data_file, config_file)
|
||||
return
|
||||
|
||||
dpath_obj = Path(data_path)
|
||||
|
||||
if dpath_obj.is_dir():
|
||||
data_files = list(dpath_obj.rglob("*.csv"))
|
||||
if not data_files:
|
||||
raise FileNotFoundError(f"No CSV files found in directory: {data_path}")
|
||||
elif dpath_obj.is_file():
|
||||
data_files = [dpath_obj]
|
||||
else:
|
||||
raise FileNotFoundError(f"Invalid data path: {data_path}")
|
||||
|
||||
if config_path is None:
|
||||
try:
|
||||
config_file = find_config_file(dpath_obj if dpath_obj.is_dir() else dpath_obj.parent)
|
||||
except FileNotFoundError:
|
||||
response = (
|
||||
input(
|
||||
"No configuration file found. Generate config file? \n"
|
||||
"You may have to modify it before processing. [y/n]: "
|
||||
)
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
|
||||
if response == "y":
|
||||
config_file = (dpath_obj if dpath_obj.is_dir() else dpath_obj.parent) / "gasflux_config.yaml"
|
||||
shutil.copy(Path(__file__).parent / "gasflux_config.yaml", config_file)
|
||||
print(f"Config file copied to: {config_file}")
|
||||
else:
|
||||
print("No config file generated. Exiting.")
|
||||
return
|
||||
else:
|
||||
config_file = Path(config_path)
|
||||
|
||||
for data_file in data_files:
|
||||
process_main(data_file, config_file)
|
||||
|
||||
|
||||
def generate_config_command(config_destination: str, recursive: bool = False, template_path: str | None = None):
|
||||
if template_path:
|
||||
template = Path(template_path)
|
||||
print(f"Using custom template: {template}")
|
||||
else:
|
||||
template = Path(__file__).parent / "gasflux_config.yaml"
|
||||
|
||||
if not template.exists():
|
||||
raise FileNotFoundError(f"Template file not found: {template}")
|
||||
|
||||
destination_path = Path(config_destination)
|
||||
|
||||
if not destination_path.is_dir():
|
||||
raise NotADirectoryError(f"Destination path is not a directory: {config_destination}")
|
||||
|
||||
overwrite_all = False # Flag to determine if we should overwrite all files without prompting
|
||||
|
||||
if recursive:
|
||||
for subdir in destination_path.rglob("*"):
|
||||
if subdir.is_dir():
|
||||
config_file = subdir / "gasflux_config.yaml"
|
||||
if config_file.exists() and not overwrite_all:
|
||||
response = (
|
||||
input(f"Config file already exists at {config_file}. Overwrite? [y/n/a/c]: ").strip().lower()
|
||||
)
|
||||
if response == "a":
|
||||
overwrite_all = True
|
||||
elif response == "c":
|
||||
print("Operation canceled.")
|
||||
return
|
||||
elif response == "n":
|
||||
print(f"Skipping {config_file}")
|
||||
continue
|
||||
shutil.copy(template, config_file)
|
||||
print(f"Config file copied to: {config_file}")
|
||||
else:
|
||||
config_file = destination_path / "gasflux_config.yaml"
|
||||
if config_file.exists() and not overwrite_all:
|
||||
response = input(f"Config file already exists at {config_file}. Overwrite? [y/n/a/c]: ").strip().lower()
|
||||
if response == "a":
|
||||
overwrite_all = True
|
||||
elif response == "c":
|
||||
print("Operation canceled.")
|
||||
return
|
||||
elif response == "n":
|
||||
print(f"Skipping {config_file}")
|
||||
return
|
||||
shutil.copy(template, config_file)
|
||||
print(f"Config file copied to: {config_file}")
|
||||
|
||||
|
||||
def main_cli():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="GasFlux Processing Pipeline",
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
add_help=False, # Disable the default help to use the custom help screen
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
|
||||
process_parser = subparsers.add_parser(
|
||||
"process",
|
||||
help="Process CSV data files",
|
||||
description=(
|
||||
"Process CSV data files located in a specified directory or a single file.\n"
|
||||
"The program will search for a configuration file (gasflux_config.yaml) in the same directory as the data file(s).\n" # noqa
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
process_parser.add_argument(
|
||||
"data_path",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Path to the data file or directory containing CSV files.\n"
|
||||
"If a directory is provided, all CSV files within the directory will be processed."
|
||||
),
|
||||
)
|
||||
process_parser.add_argument(
|
||||
"--config-path",
|
||||
"-c",
|
||||
default=None,
|
||||
help=(
|
||||
"Path to the configuration file (gasflux_config.yaml).\n"
|
||||
"If not specified, the program will search for the configuration file in the same directory as the data file(s).\n" # noqa
|
||||
"If no configuration file is found, the default configuration will be used."
|
||||
),
|
||||
)
|
||||
process_parser.add_argument(
|
||||
"--test",
|
||||
action="store_true",
|
||||
help="Use test data instead of the specified data file(s).",
|
||||
)
|
||||
|
||||
config_parser = subparsers.add_parser(
|
||||
"generate-config",
|
||||
help="Generate a configuration file",
|
||||
description=(
|
||||
"Generate a configuration file (gasflux_config.yaml) in the specified directory.\n"
|
||||
"If no directory is specified, the configuration file will be generated in the current directory."
|
||||
"Use flag --recursive to generate config files in all subdirectories recursively. YOu might want to"
|
||||
"do this if you want to have a separate config for each data file"
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
help="Generate config files in all subdirectories recursively.",
|
||||
)
|
||||
|
||||
config_parser.add_argument(
|
||||
"--template",
|
||||
default=None,
|
||||
help="Path to a custom template configuration file.",
|
||||
)
|
||||
|
||||
config_parser.add_argument(
|
||||
"config_destination",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="Destination directory for the generated configuration file.",
|
||||
)
|
||||
|
||||
# Custom help option
|
||||
parser.add_argument(
|
||||
"-h",
|
||||
"--help",
|
||||
action="store_true",
|
||||
help="Show this help message and exit",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "process":
|
||||
process_command(args.data_path, args.config_path, args.test)
|
||||
elif args.command == "generate-config":
|
||||
generate_config_command(args.config_destination, args.recursive, args.template)
|
||||
elif args.help:
|
||||
display_help()
|
||||
else:
|
||||
display_help()
|
||||
|
||||
|
||||
def display_help():
|
||||
help_text = f"""
|
||||
{Fore.CYAN}GasFlux Processing Pipeline{Style.RESET_ALL}
|
||||
{Fore.CYAN}==========================={Style.RESET_ALL}
|
||||
|
||||
{Fore.YELLOW}Description:{Style.RESET_ALL}
|
||||
The GasFlux Processing Pipeline is a command-line tool for processing CSV data files and generating configuration files.
|
||||
|
||||
{Fore.YELLOW}Usage:{Style.RESET_ALL}
|
||||
gasflux [OPTIONS] COMMAND [ARGS]...
|
||||
|
||||
{Fore.YELLOW}Commands:{Style.RESET_ALL}
|
||||
{Fore.GREEN}process{Style.RESET_ALL} Process CSV data files
|
||||
{Fore.GREEN}generate-config{Style.RESET_ALL} Generate a default configuration file
|
||||
|
||||
{Fore.YELLOW}Options:{Style.RESET_ALL}
|
||||
{Fore.BLUE}-h, --help{Style.RESET_ALL} Show this help message and exit
|
||||
{Fore.BLUE}-v, --version{Style.RESET_ALL} Show the version and exit
|
||||
|
||||
{Fore.YELLOW}Commands Help:{Style.RESET_ALL}
|
||||
{Fore.GREEN}process{Style.RESET_ALL} Process CSV data files located in a specified directory or a single file.
|
||||
The program will search for a singular configuration file (gasflux_config.yaml) in the same directory
|
||||
- if a directory is supplied to [DATA_PATH] then child directories are searched too.
|
||||
|
||||
{Fore.YELLOW}Usage:{Style.RESET_ALL} gasflux process [OPTIONS] [DATA_PATH]
|
||||
|
||||
{Fore.YELLOW}Options:{Style.RESET_ALL}
|
||||
{Fore.BLUE}-c, --config-path PATH{Style.RESET_ALL} Path to the configuration file (gasflux_config.yaml).
|
||||
If not specified, the program will search for the configuration file
|
||||
in the same directory as the data file(s).
|
||||
If no configuration file is found, the default configuration will be used.
|
||||
{Fore.BLUE}--test{Style.RESET_ALL} Use test data instead of the specified data file(s).
|
||||
{Fore.BLUE}-h, --help{Style.RESET_ALL} Show this help message and exit
|
||||
|
||||
{Fore.GREEN}generate-config{Style.RESET_ALL} Generate a default configuration file (gasflux_config.yaml) in the specified directory.
|
||||
If no directory is specified, the configuration file will be generated in the current directory.
|
||||
|
||||
{Fore.YELLOW}Usage:{Style.RESET_ALL} gasflux generate-config [OPTIONS] [CONFIG_DESTINATION]
|
||||
|
||||
{Fore.YELLOW}Options:{Style.RESET_ALL}
|
||||
{Fore.BLUE}-h, --help{Style.RESET_ALL} Show this help message and exit
|
||||
|
||||
{Fore.YELLOW}Examples:{Style.RESET_ALL}
|
||||
Process a single CSV file:
|
||||
$ gasflux process {Fore.MAGENTA}"/path/to/data.csv"{Style.RESET_ALL}
|
||||
|
||||
Process all CSV files in a directory:
|
||||
$ gasflux process {Fore.MAGENTA}"/path/to/data/directory"{Style.RESET_ALL}
|
||||
|
||||
Process a single CSV file with a specific configuration file:
|
||||
$ gasflux process {Fore.MAGENTA}"/path/to/data.csv"{Style.RESET_ALL} --config-path {Fore.MAGENTA}/path/to/config.yaml{Style.RESET_ALL}
|
||||
|
||||
Process test data:
|
||||
$ gasflux process {Fore.BLUE}--test{Style.RESET_ALL}
|
||||
|
||||
Generate a default configuration file in the current directory:
|
||||
$ gasflux generate-config {Fore.MAGENTA}.{Style.RESET_ALL}
|
||||
|
||||
Generate a default configuration file in a specific directory recursively using a custom template:
|
||||
$ gasflux generate-config {Fore.MAGENTA}"/path/to/directory"{Style.RESET_ALL} {Fore.BLUE}--recursive{Style.RESET_ALL} {Fore.BLUE}--template{Style.RESET_ALL} {Fore.MAGENTA}"/path/to/template.yaml"{Style.RESET_ALL}
|
||||
""" # noqa
|
||||
print(help_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main_cli()
|
||||
934
src/gasflux/data_processor.py
Normal file
934
src/gasflux/data_processor.py
Normal file
@ -0,0 +1,934 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
无人机数据处理脚本
|
||||
|
||||
处理Excel格式的无人机测量数据，转换为GasFlux标准输入格式。
|
||||
|
||||
支持两种使用方式：
|
||||
1. 命令行调用：python data_processor.py input.xlsx
|
||||
2. 直接调用：from data_processor import process_file; df = process_file('input.xlsx')
|
||||
|
||||
处理步骤：
|
||||
1. 读取Excel文件
|
||||
2. 删除不需要的列
|
||||
3. 根据文件名修正时间格式
|
||||
4. 坐标转换（经纬度除以10^7）
|
||||
5. 计算气压（使用qiya.py）
|
||||
6. 高度调整（减去最小高度）
|
||||
7. 时间戳融合
|
||||
8. 字段重命名
|
||||
|
||||
作者：GasFlux开发团队
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
import re
|
||||
from datetime import datetime
|
||||
import sys
|
||||
import os
|
||||
from collections import Counter
|
||||
|
||||
try:
|
||||
from tqdm import tqdm
|
||||
HAS_TQDM = True
|
||||
except ImportError:
|
||||
HAS_TQDM = False
|
||||
print("⚠️ 未安装tqdm库，将不显示进度条。如需进度条，请运行: pip install tqdm")
|
||||
|
||||
|
||||
def create_height_bins(heights, bin_size=2.0):
|
||||
"""
|
||||
将高度数据按指定间隔分档
|
||||
|
||||
Args:
|
||||
heights: 高度数据Series
|
||||
bin_size: 每个档位的间隔（米）
|
||||
|
||||
Returns:
|
||||
list: [(bin_min, bin_max, bin_center, count), ...] 每个档位的信息
|
||||
"""
|
||||
if len(heights) == 0:
|
||||
return []
|
||||
|
||||
height_min = heights.min()
|
||||
height_max = heights.max()
|
||||
|
||||
# 计算需要的档位数量
|
||||
range_size = height_max - height_min
|
||||
if range_size == 0:
|
||||
# 所有高度相同
|
||||
return [(height_min, height_max, height_min, len(heights))]
|
||||
|
||||
num_bins = max(1, int(np.ceil(range_size / bin_size)))
|
||||
|
||||
bins = []
|
||||
for i in range(num_bins):
|
||||
bin_min = height_min + i * bin_size
|
||||
bin_max = min(height_min + (i + 1) * bin_size, height_max)
|
||||
bin_center = (bin_min + bin_max) / 2
|
||||
|
||||
# 统计这个档位有多少数据点
|
||||
count = ((heights >= bin_min) & (heights < bin_max)).sum()
|
||||
if i == num_bins - 1: # 最后一个档位包含上限
|
||||
count = ((heights >= bin_min) & (heights <= bin_max)).sum()
|
||||
|
||||
if count > 0: # 只保留有数据的档位
|
||||
bins.append((bin_min, bin_max, bin_center, count))
|
||||
|
||||
return bins
|
||||
|
||||
# 导入qiya模块
|
||||
try:
|
||||
from .qiya import get_pressure_at_location
|
||||
print("✅ 成功导入qiya模块")
|
||||
except ImportError as e:
|
||||
print(f"❌ 导入qiya模块失败: {e}")
|
||||
print("请确保GasFlux包结构完整")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def load_excel_data(file_path):
|
||||
"""
|
||||
读取Excel文件并进行初步处理
|
||||
|
||||
Args:
|
||||
file_path: Excel文件路径
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 读取的数据
|
||||
"""
|
||||
try:
|
||||
print(f"正在读取文件: {file_path}")
|
||||
|
||||
# 读取Excel文件
|
||||
df = pd.read_excel(file_path)
|
||||
print(f"✅ 成功读取数据：{len(df)} 行，{len(df.columns)} 列")
|
||||
print(f"列名：{list(df.columns)}")
|
||||
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 读取文件失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def remove_columns(df, columns_to_remove):
|
||||
"""
|
||||
删除指定的列
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
columns_to_remove: 要删除的列名列表
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 删除列后的DataFrame
|
||||
"""
|
||||
print(f"\n删除列: {columns_to_remove}")
|
||||
|
||||
# 检查要删除的列是否存在
|
||||
existing_columns = [col for col in columns_to_remove if col in df.columns]
|
||||
missing_columns = [col for col in columns_to_remove if col not in df.columns]
|
||||
|
||||
if missing_columns:
|
||||
print(f"⚠️ 以下列不存在（跳过）: {missing_columns}")
|
||||
|
||||
if existing_columns:
|
||||
df = df.drop(columns=existing_columns)
|
||||
print(f"✅ 已删除 {len(existing_columns)} 列")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def extract_hour_from_filename(filename):
|
||||
"""
|
||||
从文件名中提取小时信息
|
||||
|
||||
例如: "08_34_01_间隔高度5m.xlsx" -> "08"
|
||||
|
||||
Args:
|
||||
filename: 文件名
|
||||
|
||||
Returns:
|
||||
str: 小时字符串（两位数）
|
||||
"""
|
||||
# 使用正则表达式匹配小时部分
|
||||
match = re.match(r'(\d{2})_', filename)
|
||||
if match:
|
||||
return match.group(1)
|
||||
else:
|
||||
print(f"⚠️ 无法从文件名 '{filename}' 中提取小时信息，使用默认值 '00'")
|
||||
return "00"
|
||||
|
||||
|
||||
def fix_time_column(df, filename):
|
||||
"""
|
||||
根据文件名修正时间列
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
filename: 文件名
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 修正后的DataFrame
|
||||
"""
|
||||
print("修正时间格式...")
|
||||
|
||||
# 提取小时信息
|
||||
hour_prefix = extract_hour_from_filename(filename)
|
||||
print(f"从文件名提取的小时: {hour_prefix}")
|
||||
|
||||
# 检查时间列是否存在
|
||||
if '时间' not in df.columns:
|
||||
print("❌ 未找到 '时间' 列")
|
||||
return df
|
||||
|
||||
# 修正时间格式
|
||||
def fix_single_time(time_str):
|
||||
if pd.isna(time_str):
|
||||
return time_str
|
||||
|
||||
time_str = str(time_str).strip()
|
||||
|
||||
# 如果时间格式类似 "0:34:01" 或 "00:34:01"
|
||||
if re.match(r'^\d{1,2}:\d{2}:\d{2}$', time_str):
|
||||
parts = time_str.split(':')
|
||||
original_hour = int(parts[0])
|
||||
filename_hour = int(hour_prefix)
|
||||
|
||||
# 小时相加
|
||||
new_hour = (filename_hour + original_hour) % 24 # 防止超过24小时
|
||||
|
||||
# 保持分钟和秒不变
|
||||
new_time = f"{new_hour:02d}:{parts[1]}:{parts[2]}"
|
||||
return new_time
|
||||
else:
|
||||
# 其他格式，使用默认时间
|
||||
return f"{hour_prefix}:00:00"
|
||||
|
||||
# 应用时间修正
|
||||
original_times = df['时间'].head(3).tolist()
|
||||
df['时间'] = df['时间'].apply(fix_single_time)
|
||||
corrected_times = df['时间'].head(3).tolist()
|
||||
|
||||
print(f"时间修正示例:")
|
||||
for orig, corr in zip(original_times, corrected_times):
|
||||
print(f" {orig} → {corr}")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def convert_coordinates(df):
|
||||
"""
|
||||
转换经纬度坐标（除以10^7）
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 转换后的DataFrame
|
||||
"""
|
||||
print("转换经纬度坐标...")
|
||||
|
||||
if '经度' in df.columns:
|
||||
original_lon = df['经度'].head(3).tolist()
|
||||
df['经度'] = df['经度'] / 1e7
|
||||
converted_lon = df['经度'].head(3).tolist()
|
||||
print("经度转换示例:")
|
||||
for orig, conv in zip(original_lon, converted_lon):
|
||||
print(".6f")
|
||||
|
||||
if '纬度' in df.columns:
|
||||
original_lat = df['纬度'].head(3).tolist()
|
||||
df['纬度'] = df['纬度'] / 1e7
|
||||
converted_lat = df['纬度'].head(3).tolist()
|
||||
print("纬度转换示例:")
|
||||
for orig, conv in zip(original_lat, converted_lat):
|
||||
print(".6f")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def calculate_pressure(df, max_samples=None, height_tolerance=10.0, height_bin_size=2.0):
|
||||
"""
|
||||
计算气压数据（高度分档优化版）
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
max_samples: 最大采样数量（None表示计算所有行）
|
||||
height_tolerance: 高度变化容差（米），如果所有高度都在此范围内，只计算一次
|
||||
height_bin_size: 高度分档间隔（米），每个档位使用中间高度计算气压
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 添加气压列的DataFrame
|
||||
"""
|
||||
print("计算气压数据...")
|
||||
|
||||
# 检查必要列是否存在
|
||||
required_cols = ['日期', '时间', '经度', '纬度', '融合高程']
|
||||
missing_cols = [col for col in required_cols if col not in df.columns]
|
||||
|
||||
if missing_cols:
|
||||
print(f"❌ 缺少必要列: {missing_cols}")
|
||||
return df
|
||||
|
||||
# 检查高度变化范围
|
||||
height_min = df['融合高程'].min()
|
||||
height_max = df['融合高程'].max()
|
||||
height_range = height_max - height_min
|
||||
|
||||
print(f"🏔️ 高度范围: {height_min:.1f} - {height_max:.1f} 米 (变化: {height_range:.1f} 米)")
|
||||
# 创建高度分档
|
||||
height_bins = create_height_bins(df['融合高程'], height_bin_size)
|
||||
print(f"📏 高度分档: {len(height_bins)} 个档位 (间隔: {height_bin_size:.1f} 米)")
|
||||
|
||||
for i, (bin_min, bin_max, bin_center, count) in enumerate(height_bins):
|
||||
print(f" 档位{i+1}: {bin_min:.1f}-{bin_max:.1f}m (中心: {bin_center:.1f}m, 数据: {count}行)")
|
||||
# 决定计算策略
|
||||
if height_range <= height_tolerance:
|
||||
# 高度变化小，只计算一次气压
|
||||
print("🎯 高度变化小，将使用平均高度计算一次气压")
|
||||
use_single_calculation = True
|
||||
mean_height = df['融合高程'].mean()
|
||||
print(f"📍 使用平均高度: {mean_height:.1f} 米")
|
||||
elif len(height_bins) == 1:
|
||||
# 只有一个高度档位，使用档位中心高度
|
||||
print("📦 只有一个高度档位，使用档位中心高度")
|
||||
use_single_calculation = True
|
||||
mean_height = height_bins[0][2] # bin_center
|
||||
print(f"📍 使用档位中心高度: {mean_height:.1f} 米")
|
||||
else:
|
||||
# 高度变化大，使用分档计算
|
||||
print("🏗️ 使用高度分档策略，减少API调用")
|
||||
use_single_calculation = False
|
||||
|
||||
# 确定要处理的行数
|
||||
if max_samples is None or max_samples >= len(df):
|
||||
# 计算所有行
|
||||
sample_df = df.copy()
|
||||
actual_samples = len(df)
|
||||
if not use_single_calculation:
|
||||
print(f"📊 将计算所有 {len(df)} 行的气压数据")
|
||||
else:
|
||||
# 限制采样数量
|
||||
print(f"⚠️ 数据量较大 ({len(df)} 行)，只对前 {max_samples} 行计算气压")
|
||||
sample_df = df.head(max_samples).copy()
|
||||
actual_samples = max_samples
|
||||
|
||||
pressures = []
|
||||
|
||||
if use_single_calculation:
|
||||
# 只计算一次气压
|
||||
try:
|
||||
# 使用第一行的日期和时间作为代表
|
||||
first_row = sample_df.iloc[0]
|
||||
|
||||
# 转换日期格式 - 只提取日期部分，移除任何时间信息
|
||||
date_str = str(first_row['日期'])
|
||||
if ' ' in date_str:
|
||||
date_str = date_str.split(' ')[0]
|
||||
elif 'T' in date_str:
|
||||
date_str = date_str.split('T')[0]
|
||||
|
||||
if '/' in date_str:
|
||||
date_str = date_str.replace('/', '-')
|
||||
|
||||
date_parts = date_str.split('-')
|
||||
if len(date_parts) == 3 and len(date_parts[0]) == 4:
|
||||
year, month, day = date_parts
|
||||
formatted_date = f"{year}-{month.zfill(2)}-{day.zfill(2)}"
|
||||
else:
|
||||
raise ValueError(f"日期格式异常: {date_str}")
|
||||
|
||||
# 使用数据的代表性时间（整点小时，众数）
|
||||
time_strings = []
|
||||
for time_val in sample_df['时间']:
|
||||
time_str = str(time_val).strip()
|
||||
if ':' in time_str:
|
||||
# 确保是有效的 HH:MM 格式，然后取整点小时
|
||||
parts = time_str.split(':')
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
hour = int(parts[0])
|
||||
# 确保小时在有效范围内 (0-23)
|
||||
if 0 <= hour <= 23:
|
||||
time_strings.append(f"{hour:02d}:00")
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if time_strings:
|
||||
time_counts = Counter(time_strings)
|
||||
formatted_time = time_counts.most_common(1)[0][0]
|
||||
occurrence_count = time_counts.most_common(1)[0][1]
|
||||
print(f"数据时间选择: {formatted_time} ({occurrence_count}/{len(time_strings)} 次，{occurrence_count/len(time_strings)*100:.1f}%)")
|
||||
else:
|
||||
formatted_time = "12:00"
|
||||
print("无有效时间数据，使用默认中午12:00")
|
||||
|
||||
print("正在计算平均气压...")
|
||||
pressure = get_pressure_at_location(
|
||||
lat=sample_df['纬度'].mean(),
|
||||
lon=sample_df['经度'].mean(),
|
||||
altitude=mean_height,
|
||||
date=formatted_date,
|
||||
time=formatted_time
|
||||
)
|
||||
|
||||
if pressure is not None:
|
||||
pressures = [pressure] * len(sample_df)
|
||||
print("✅ 平均气压计算成功，将应用到所有行")
|
||||
else:
|
||||
print("❌ 平均气压计算失败")
|
||||
pressures = [None] * len(sample_df)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 平均气压计算失败: {e}")
|
||||
pressures = [None] * len(sample_df)
|
||||
|
||||
else:
|
||||
# 使用高度分档策略
|
||||
print("🏗️ 开始分档计算气压...")
|
||||
|
||||
# 为每个高度档位计算气压
|
||||
bin_pressures = {} # bin_center -> pressure
|
||||
|
||||
# 设置进度条
|
||||
iterator = height_bins
|
||||
if HAS_TQDM:
|
||||
iterator = tqdm(iterator, total=len(height_bins), desc="计算气压档位", unit="档")
|
||||
|
||||
for bin_min, bin_max, bin_center, count in iterator:
|
||||
try:
|
||||
# 使用第一行数据作为代表来获取日期和时间
|
||||
# 找到这个档位中的一行数据
|
||||
bin_rows = sample_df[(sample_df['融合高程'] >= bin_min) &
|
||||
(sample_df['融合高程'] <= bin_max)]
|
||||
if len(bin_rows) == 0:
|
||||
continue
|
||||
|
||||
first_row = bin_rows.iloc[0]
|
||||
|
||||
# 转换日期格式 - 只提取日期部分，移除任何时间信息
|
||||
date_str = str(first_row['日期'])
|
||||
if ' ' in date_str:
|
||||
date_str = date_str.split(' ')[0]
|
||||
elif 'T' in date_str:
|
||||
date_str = date_str.split('T')[0]
|
||||
|
||||
if '/' in date_str:
|
||||
date_str = date_str.replace('/', '-')
|
||||
|
||||
date_parts = date_str.split('-')
|
||||
if len(date_parts) == 3 and len(date_parts[0]) == 4:
|
||||
year, month, day = date_parts
|
||||
formatted_date = f"{year}-{month.zfill(2)}-{day.zfill(2)}"
|
||||
else:
|
||||
print(f"⚠️ 档位高度 {bin_center:.1f}m 日期格式异常: {date_str}")
|
||||
bin_pressures[bin_center] = None
|
||||
continue
|
||||
|
||||
# 使用该档位数据的代表性时间（整点小时）
|
||||
time_strings = []
|
||||
for time_val in bin_rows['时间']:
|
||||
time_str = str(time_val).strip()
|
||||
if ':' in time_str:
|
||||
# 确保是有效的 HH:MM 格式，然后取整点小时
|
||||
parts = time_str.split(':')
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
hour = int(parts[0])
|
||||
# 确保小时在有效范围内 (0-23)
|
||||
if 0 <= hour <= 23:
|
||||
time_strings.append(f"{hour:02d}:00")
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if time_strings:
|
||||
# 使用最常见的时间（众数）
|
||||
time_counts = Counter(time_strings)
|
||||
formatted_time = time_counts.most_common(1)[0][0]
|
||||
occurrence_count = time_counts.most_common(1)[0][1]
|
||||
print(f" 档位时间选择: {formatted_time} ({occurrence_count}/{len(time_strings)} 次，{occurrence_count/len(time_strings)*100:.1f}%)")
|
||||
else:
|
||||
# 如果没有有效时间，使用中午12点
|
||||
formatted_time = "12:00"
|
||||
print(" 无有效时间数据，使用默认中午12:00")
|
||||
|
||||
# 计算这个档位的气压（使用平均位置和档位中心高度）
|
||||
avg_lat = bin_rows['纬度'].mean()
|
||||
avg_lon = bin_rows['经度'].mean()
|
||||
|
||||
pressure = get_pressure_at_location(
|
||||
lat=avg_lat,
|
||||
lon=avg_lon,
|
||||
altitude=bin_center,
|
||||
date=formatted_date,
|
||||
time=formatted_time
|
||||
)
|
||||
|
||||
bin_pressures[bin_center] = pressure
|
||||
|
||||
# 更新进度条
|
||||
if HAS_TQDM:
|
||||
success_count = sum(1 for p in bin_pressures.values() if p is not None)
|
||||
iterator.set_description(f"计算档位 (成功: {success_count}/{len(bin_pressures)})")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 计算高度档位 {bin_center:.1f}m 气压失败: {e}")
|
||||
bin_pressures[bin_center] = None
|
||||
|
||||
# 为每一行分配对应档位的气压
|
||||
pressures = []
|
||||
for idx, row in sample_df.iterrows():
|
||||
# 找到这个高度对应的档位
|
||||
height = row['融合高程']
|
||||
assigned_pressure = None
|
||||
|
||||
for bin_min, bin_max, bin_center, count in height_bins:
|
||||
if bin_min <= height <= bin_max:
|
||||
assigned_pressure = bin_pressures.get(bin_center)
|
||||
break
|
||||
|
||||
pressures.append(assigned_pressure)
|
||||
|
||||
print(f"✅ 完成分档气压计算，共 {len(bin_pressures)} 个档位，{len([p for p in bin_pressures.values() if p is not None])} 个成功")
|
||||
|
||||
# 添加气压列
|
||||
df['pressure'] = None # 初始化
|
||||
df.loc[sample_df.index, 'pressure'] = pressures
|
||||
|
||||
# 对于未计算的行，使用插值或平均值填充
|
||||
if max_samples is not None and len(df) > max_samples:
|
||||
# 只计算了部分行，用平均值填充其余行
|
||||
valid_pressures_for_mean = [p for p in pressures if p is not None]
|
||||
if valid_pressures_for_mean:
|
||||
mean_pressure = sum(valid_pressures_for_mean) / len(valid_pressures_for_mean)
|
||||
df['pressure'] = df['pressure'].fillna(mean_pressure)
|
||||
print(f"使用平均气压填充其余 {len(df) - max_samples} 行: {mean_pressure:.1f} hPa")
|
||||
|
||||
# 统计信息
|
||||
valid_pressures = [p for p in pressures if p is not None]
|
||||
if valid_pressures:
|
||||
avg_pressure = sum(valid_pressures) / len(valid_pressures)
|
||||
print(f"成功计算 {len(valid_pressures)}/{actual_samples} 个气压值，平均值: {avg_pressure:.1f} hPa")
|
||||
else:
|
||||
print("⚠️ 未能计算出任何气压值")
|
||||
return df
|
||||
|
||||
|
||||
def adjust_altitude(df):
|
||||
"""
|
||||
调整融合高程（减去最小值）
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 调整后的DataFrame
|
||||
"""
|
||||
print("调整融合高程...")
|
||||
|
||||
if '融合高程' in df.columns:
|
||||
min_altitude = df['融合高程'].min()
|
||||
print(".2f")
|
||||
|
||||
original_alt = df['融合高程'].head(3).tolist()
|
||||
df['融合高程'] = df['融合高程'] - min_altitude
|
||||
adjusted_alt = df['融合高程'].head(3).tolist()
|
||||
|
||||
print("高度调整示例:")
|
||||
for orig, adj in zip(original_alt, adjusted_alt):
|
||||
print(".2f")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def merge_timestamp(df):
|
||||
"""
|
||||
融合日期和时间列为时间戳
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 融合后的DataFrame
|
||||
"""
|
||||
print("融合日期和时间...")
|
||||
|
||||
if '日期' in df.columns and '时间' in df.columns:
|
||||
timestamps = []
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
try:
|
||||
date_str = str(row['日期'])
|
||||
time_str = str(row['时间'])
|
||||
|
||||
# 清理日期字符串 - 移除任何时间部分
|
||||
date_str = date_str.strip()
|
||||
if ' ' in date_str:
|
||||
date_str = date_str.split(' ')[0] # 只取日期部分
|
||||
if 'T' in date_str:
|
||||
date_str = date_str.split('T')[0] # 处理ISO格式
|
||||
|
||||
# 标准化日期格式
|
||||
if '/' in date_str:
|
||||
date_str = date_str.replace('/', '-')
|
||||
|
||||
# 确保日期格式正确
|
||||
date_parts = date_str.split('-')
|
||||
if len(date_parts) == 3:
|
||||
year, month, day = date_parts
|
||||
date_formatted = f"{year.zfill(4)}-{month.zfill(2)}-{day.zfill(2)}"
|
||||
else:
|
||||
print(f"⚠️ 日期格式异常: '{date_str}'，使用当前日期")
|
||||
date_formatted = datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
# 时间字符串已经是修正后的格式（如 "08:34:01"），直接使用
|
||||
time_str = time_str.strip()
|
||||
if ':' in time_str and len(time_str.split(':')) >= 2:
|
||||
time_formatted = time_str
|
||||
else:
|
||||
print(f"⚠️ 时间格式异常: '{time_str}'，使用默认时间")
|
||||
time_formatted = "12:00:00"
|
||||
|
||||
# 组合时间戳 - 直接连接日期和时间
|
||||
timestamp = f"{date_formatted} {time_formatted}"
|
||||
timestamps.append(timestamp)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 处理第 {idx+1} 行时间戳失败: {e}")
|
||||
timestamps.append(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
|
||||
df['timestamp'] = timestamps
|
||||
|
||||
print("时间戳融合示例:")
|
||||
for i in range(min(3, len(timestamps))):
|
||||
print(f" {df.loc[i, '日期']} + {df.loc[i, '时间']} → {timestamps[i]}")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def rename_columns(df):
|
||||
"""
|
||||
重命名字段为GasFlux标准格式
|
||||
|
||||
Args:
|
||||
df: 输入DataFrame
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 重命名后的DataFrame
|
||||
"""
|
||||
print("重命名字段...")
|
||||
|
||||
# 定义字段映射
|
||||
column_mapping = {
|
||||
'timestamp': 'timestamp', # 时间戳（已创建）
|
||||
'经度': 'longitude', # 经度 → latitude
|
||||
'纬度': 'latitude', # 纬度 → longitude
|
||||
'融合高程': 'height_ato', # 融合高程 → height_ato
|
||||
'修正风向': 'winddir', # 修正风向 → winddir
|
||||
'修正风速': 'windspeed', # 修正风速 → windspeed
|
||||
'风温': 'temperature', # 风温 → temperature
|
||||
'pressure': 'pressure', # 气压（已计算）
|
||||
'CH4': 'ch4', # CH4保持不变
|
||||
'pitch': 'course_elevation', # pitch → course_elevation
|
||||
'yaw': 'course_azimuth' # yaw → course_azimuth
|
||||
}
|
||||
|
||||
# 重命名存在的列
|
||||
columns_to_rename = {}
|
||||
for old_name, new_name in column_mapping.items():
|
||||
if old_name in df.columns:
|
||||
columns_to_rename[old_name] = new_name
|
||||
|
||||
if columns_to_rename:
|
||||
df = df.rename(columns=columns_to_rename)
|
||||
print("字段重命名:")
|
||||
for old, new in columns_to_rename.items():
|
||||
print(f" {old} → {new}")
|
||||
|
||||
# 只保留GasFlux需要的列
|
||||
required_columns = ['timestamp', 'latitude', 'longitude', 'height_ato', 'windspeed', 'winddir', 'temperature', 'pressure', 'ch4', 'course_elevation', 'course_azimuth']
|
||||
existing_required_columns = [col for col in required_columns if col in df.columns]
|
||||
|
||||
if len(existing_required_columns) != len(required_columns):
|
||||
missing = [col for col in required_columns if col not in df.columns]
|
||||
print(f"⚠️ 缺少必需列: {missing}")
|
||||
|
||||
# 移除不需要的列，只保留必需的列
|
||||
df = df[existing_required_columns]
|
||||
print(f"最终保留列: {existing_required_columns}")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def process_excel_file(file_path):
|
||||
"""
|
||||
处理单个Excel文件的主函数
|
||||
|
||||
Args:
|
||||
file_path: Excel文件路径
|
||||
"""
|
||||
print(f"=== 开始处理文件: {file_path} ===\n")
|
||||
|
||||
# 获取文件名（用于时间修正）
|
||||
filename = Path(file_path).name
|
||||
|
||||
# 1. 读取数据
|
||||
df = load_excel_data(file_path)
|
||||
|
||||
# 2. 删除不需要的列
|
||||
columns_to_remove = [
|
||||
'高程', '速度x', '速度y', '速度z',
|
||||
'四元数_q0', '四元数_q1', '四元数_q2', '四元数_q3',
|
||||
'roll', 'H2O', # 保留pitch和yaw，将重命名为course_elevation和course_azimuth
|
||||
'原始风向', '原始风速'
|
||||
]
|
||||
df = remove_columns(df, columns_to_remove)
|
||||
|
||||
# 3. 修正时间格式
|
||||
df = fix_time_column(df, filename)
|
||||
|
||||
# 4. 坐标转换
|
||||
df = convert_coordinates(df)
|
||||
|
||||
# 5. 计算气压
|
||||
df = calculate_pressure(df, max_samples=None, height_tolerance=10.0, height_bin_size=2.0) # 计算所有行，高度容差10米，分档2米
|
||||
|
||||
# 6. 高度调整
|
||||
df = adjust_altitude(df)
|
||||
|
||||
# 7. 时间戳融合
|
||||
df = merge_timestamp(df)
|
||||
|
||||
# 调试：检查当前列
|
||||
print(f"时间戳融合后列名: {list(df.columns)}")
|
||||
if 'timestamp' in df.columns:
|
||||
print(f"timestamp列示例: {df['timestamp'].head(3).tolist()}")
|
||||
|
||||
# 8. 字段重命名
|
||||
df = rename_columns(df)
|
||||
|
||||
# 保存处理结果
|
||||
output_path = Path(file_path).with_suffix('.processed.csv')
|
||||
df.to_csv(output_path, index=False)
|
||||
|
||||
print(f"\n✅ 处理完成！")
|
||||
print(f"📁 输出文件: {output_path}")
|
||||
print(f"📊 最终数据形状: {df.shape[0]} 行 × {df.shape[1]} 列")
|
||||
print(f"📋 最终列名: {list(df.columns)}")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def process_file(input_file, output_file=None):
|
||||
"""
|
||||
直接处理Excel文件的函数（不使用命令行参数）
|
||||
|
||||
Args:
|
||||
input_file: 输入Excel文件路径（字符串或Path对象）
|
||||
output_file: 输出CSV文件路径（可选，字符串或Path对象）
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 处理后的DataFrame
|
||||
"""
|
||||
# 转换为Path对象
|
||||
input_path = Path(input_file)
|
||||
|
||||
# 检查输入文件
|
||||
if not input_path.exists():
|
||||
raise FileNotFoundError(f"输入文件不存在: {input_path}")
|
||||
|
||||
if input_path.suffix.lower() not in ['.xlsx', '.xls']:
|
||||
raise ValueError(f"输入文件必须是Excel格式 (.xlsx 或 .xls)，当前文件: {input_path}")
|
||||
|
||||
# 处理文件
|
||||
df = process_excel_file(str(input_path))
|
||||
|
||||
# 如果指定了输出路径，额外保存一份
|
||||
if output_file:
|
||||
output_path = Path(output_file)
|
||||
df.to_csv(output_path, index=False)
|
||||
print(f"📁 额外保存到: {output_path}")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def interactive_input():
|
||||
"""
|
||||
交互式输入模式 - 手动输入参数
|
||||
|
||||
Returns:
|
||||
tuple: (input_file, output_file) 文件路径元组
|
||||
"""
|
||||
print("=== 手动输入模式 ===")
|
||||
print("请按照提示输入参数...\n")
|
||||
|
||||
# 输入文件路径
|
||||
while True:
|
||||
input_file = input("请输入Excel文件路径 (例如: data.xlsx): ").strip()
|
||||
if not input_file:
|
||||
print("❌ 文件路径不能为空，请重新输入")
|
||||
continue
|
||||
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"❌ 文件不存在: {input_path}")
|
||||
print("提示: 请确保文件路径正确，或者将文件放在当前目录下")
|
||||
continue
|
||||
|
||||
if input_path.suffix.lower() not in ['.xlsx', '.xls']:
|
||||
print(f"❌ 文件格式错误: {input_path.suffix}")
|
||||
print("只支持 .xlsx 和 .xls 格式的Excel文件")
|
||||
continue
|
||||
|
||||
break
|
||||
|
||||
# 输出文件路径（可选）
|
||||
output_file = input("请输入输出CSV文件路径 (可选，直接回车使用默认): ").strip()
|
||||
if not output_file:
|
||||
output_file = None
|
||||
print("使用默认输出文件名")
|
||||
|
||||
print(f"\n✅ 输入确认:")
|
||||
print(f" 输入文件: {input_file}")
|
||||
print(f" 输出文件: {output_file or '自动生成'}")
|
||||
|
||||
confirm = input("\n确认开始处理? (y/N): ").strip().lower()
|
||||
if confirm not in ['y', 'yes', '是', '确认']:
|
||||
print("❌ 用户取消操作")
|
||||
return None, None
|
||||
|
||||
return input_file, output_file
|
||||
|
||||
|
||||
def main(input_file=None, output_file=None, interactive=False):
|
||||
"""
|
||||
主函数 - 支持多种输入方式
|
||||
|
||||
Args:
|
||||
input_file: 输入文件路径（直接调用时使用）
|
||||
output_file: 输出文件路径（直接调用时使用）
|
||||
interactive: 是否启用交互式输入模式
|
||||
"""
|
||||
# 如果启用交互式输入
|
||||
if interactive:
|
||||
input_file, output_file = interactive_input()
|
||||
if input_file is None:
|
||||
return None
|
||||
|
||||
# 如果提供了直接参数，使用直接参数
|
||||
if input_file is not None:
|
||||
try:
|
||||
return process_file(input_file, output_file)
|
||||
except Exception as e:
|
||||
print(f"❌ 处理失败: {e}")
|
||||
raise
|
||||
|
||||
# 否则使用命令行参数
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="无人机数据处理工具 - 将Excel数据转换为GasFlux格式",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
使用方式:
|
||||
|
||||
1. 命令行模式:
|
||||
python data_processor.py data.xlsx
|
||||
python data_processor.py data.xlsx -o output.csv
|
||||
|
||||
2. 交互式模式:
|
||||
python data_processor.py --interactive
|
||||
|
||||
3. 直接调用:
|
||||
from data_processor import process_file
|
||||
df = process_file('data.xlsx', 'output.csv')
|
||||
|
||||
4. Python脚本调用:
|
||||
from data_processor import main
|
||||
df = main(input_file='data.xlsx', output_file='output.csv')
|
||||
|
||||
处理步骤:
|
||||
1. 读取Excel文件
|
||||
2. 删除不需要的列
|
||||
3. 根据文件名修正时间格式
|
||||
4. 经纬度坐标转换 (除以10^7)
|
||||
5. 计算气压数据
|
||||
6. 高度调整 (减去最小值)
|
||||
7. 时间戳融合
|
||||
8. 字段重命名为GasFlux格式
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('input_file', nargs='?', help='输入的Excel文件路径')
|
||||
parser.add_argument('-o', '--output', help='输出CSV文件路径（可选，默认自动生成）')
|
||||
parser.add_argument('-i', '--interactive', action='store_true', help='启用交互式输入模式')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 如果启用交互式模式
|
||||
if args.interactive:
|
||||
input_file, output_file = interactive_input()
|
||||
if input_file is None:
|
||||
return None
|
||||
else:
|
||||
input_file = args.input_file
|
||||
output_file = args.output
|
||||
|
||||
# 如果没有提供输入文件，显示帮助和使用示例
|
||||
if not input_file:
|
||||
parser.print_help()
|
||||
print("\n" + "="*60)
|
||||
print("📖 使用示例:")
|
||||
print("="*60)
|
||||
print("1. 命令行模式:")
|
||||
print(" python data_processor.py your_file.xlsx")
|
||||
print(" python data_processor.py data.xlsx -o output.csv")
|
||||
print("")
|
||||
print("2. Python脚本中直接调用:")
|
||||
print(" from data_processor import process_file")
|
||||
print(" df = process_file('input.xlsx')")
|
||||
print(" df = process_file('input.xlsx', 'output.csv')")
|
||||
print("")
|
||||
print("3. 交互式模式:")
|
||||
print(" python data_processor.py --interactive")
|
||||
print("="*60)
|
||||
return None
|
||||
|
||||
# 检查输入文件
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"❌ 错误：输入文件不存在: {input_path}")
|
||||
sys.exit(1)
|
||||
|
||||
if input_path.suffix.lower() not in ['.xlsx', '.xls']:
|
||||
print(f"❌ 错误：输入文件必须是Excel格式 (.xlsx 或 .xls)")
|
||||
sys.exit(1)
|
||||
|
||||
# 处理文件
|
||||
try:
|
||||
df = process_excel_file(str(input_path))
|
||||
|
||||
# 如果指定了输出路径，额外保存一份
|
||||
if output_file:
|
||||
output_path = Path(output_file)
|
||||
df.to_csv(output_path, index=False)
|
||||
print(f"📁 额外保存到: {output_path}")
|
||||
|
||||
return df
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n⚠️ 用户中断处理")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n❌ 处理失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 当直接运行脚本时，使用命令行参数模式
|
||||
main()
|
||||
58
src/gasflux/gas.py
Normal file
58
src/gasflux/gas.py
Normal file
@ -0,0 +1,58 @@
|
||||
"""functions related to gas transformations and calculations, e.g. density, point flux etc."""
|
||||
|
||||
import molmass
|
||||
import pandas as pd
|
||||
|
||||
gas_variables = {
|
||||
"standard_pressure": 1013.25, # hPa/mbar
|
||||
"standard_temperature": 273.15, # degrees K
|
||||
"standard_molar_volume": 0.022413969545014, # m3⋅mol-1
|
||||
}
|
||||
|
||||
|
||||
def mass(formula: str) -> float:
|
||||
"""Return the molar mass of a gas in g/mol."""
|
||||
return molmass.Formula(formula.upper()).mass # only accepts capital letters
|
||||
|
||||
|
||||
def gas_density(local_pressure: float, local_temperature_celsius: float, gas: str) -> float: # millibars and celsius
|
||||
"""
|
||||
Calculate the density of a gas in kg/m3 based on local pressure and temperature.
|
||||
|
||||
Parameters:
|
||||
- local_pressure: The local pressure in hPa/mbar.
|
||||
- local_temperature: The local temperature in degrees Celsius.
|
||||
- gas: The chemical formula of the gas.
|
||||
|
||||
Returns:
|
||||
- The density of the gas in kg/m3.
|
||||
|
||||
Assumes ideal gas behavior.
|
||||
"""
|
||||
local_temperature_kelvin = local_temperature_celsius + gas_variables["standard_temperature"]
|
||||
local_volume = (
|
||||
gas_variables["standard_molar_volume"]
|
||||
* (gas_variables["standard_pressure"] / local_pressure)
|
||||
* ((local_temperature_kelvin + gas_variables["standard_temperature"]) / gas_variables["standard_temperature"])
|
||||
) # m3⋅mol-1
|
||||
return mass(gas) / 1000 / local_volume
|
||||
|
||||
|
||||
def gas_flux_column(df: pd.DataFrame, gas: str, wind: str = "windspeed") -> pd.DataFrame:
|
||||
"""
|
||||
Add columns to the DataFrame for the gas density, mass, and flux.
|
||||
|
||||
Parameters:
|
||||
- df: The DataFrame.
|
||||
- gas: The chemical formula of the gas.
|
||||
- wind: The column name for the wind speed (NB - must be perpendicular to the plane)
|
||||
|
||||
Returns:
|
||||
- The DataFrame with the added columns.
|
||||
"""
|
||||
average_temp = df["temperature"].mean() # celsius
|
||||
average_pressure = df["pressure"].mean() # hPa
|
||||
gd = gas_density(local_pressure=average_pressure, local_temperature_celsius=average_temp, gas=gas) # kg/m3
|
||||
df[f"{gas}_kg_m3"] = gd * (df[f"{gas}_normalised"] * 1e-6) # kg/m3
|
||||
df[f"{gas}_kg_h_m2"] = df[f"{gas}_kg_m3"] * df[wind] * 60 * 60 # kg/h/m2
|
||||
return df
|
||||
69
src/gasflux/gasflux_config.yaml
Normal file
69
src/gasflux/gasflux_config.yaml
Normal file
@ -0,0 +1,69 @@
|
||||
# gasflux_config.yaml
|
||||
|
||||
output_dir: ~/gasflux_reports
|
||||
|
||||
# required columns and maximum ranges
|
||||
required_cols:
|
||||
latitude: [-90, 90]
|
||||
longitude: [-180, 180]
|
||||
height_ato: [-200, 500] # meters above takeoff
|
||||
windspeed: [0, 50] # m/s
|
||||
winddir: [0, 360] # degrees
|
||||
temperature: [-50, 60] # degrees Celsius
|
||||
pressure: [900, 1100] # hPa/mb
|
||||
|
||||
# optional gas columns and maximum ppm ranges. Relative concentrations are used so offset can be wrong as long as gain and linearity are correct.
|
||||
gases:
|
||||
ch4: [1.5, 500]
|
||||
co2: [300, 5000]
|
||||
c2h6: [-0.5, 10]
|
||||
|
||||
strategies:
|
||||
background: "algorithm" # currently only algorithm (via pybaselines) is supported
|
||||
sensor: "insitu" # currently only insitu is supported
|
||||
spatial: "curtain" # currently "curtain" and "spiral" are supported
|
||||
interpolation: "kriging" # currently only kriging is supported
|
||||
|
||||
# baseline settings.
|
||||
algorithmic_baseline_settings:
|
||||
algorithm: fastchrom
|
||||
fastchrom: {
|
||||
"half_window": 6,
|
||||
"threshold": "custom", #
|
||||
"min_fwhm": ~,
|
||||
"interp_half_window": 3,
|
||||
"smooth_half_window": 3,
|
||||
"weights": ~,
|
||||
"max_iter": 100,
|
||||
"min_length": 2}
|
||||
fabc : {
|
||||
"lam": 10000, # The smoothing parameter. Larger values will create smoother baselines. Default is 1e6.
|
||||
"scale": 10, # The scale at which to calculate the continuous wavelet transform. Should be approximately equal to the index-based full-width-at-half-maximum of the peaks or features in the data. Default is None, which will use half of the value from optimize_window(), which is not always a good value, but at least scales with the number of data points and gives a starting point for tuning the parameter.
|
||||
"diff_order": 2} # The order of the differential matrix. Must be greater than 0. Default is 2 (second order differential matrix). Typical values are 2 or 1.
|
||||
dietrich : {
|
||||
"poly_order": 5,
|
||||
"smooth_half_window": 5,}
|
||||
golotvin : {
|
||||
"half_window": 2,
|
||||
"sections": 10}
|
||||
|
||||
# kriging settings
|
||||
semivariogram_settings:
|
||||
model: spherical
|
||||
estimator: cressie
|
||||
n_lags: 20
|
||||
bin_func: even
|
||||
fit_method: lm
|
||||
### Size of the search window; if the algorithm complains about not having enough neighbours, consider increasing these
|
||||
maxlag: 100 # in meters.
|
||||
tolerance: 10 # in degrees
|
||||
azimuth: 0 # in degrees (0 is right/horizontal)
|
||||
bandwidth: 20 # in meters
|
||||
# fit_sigma: linear # this should allow for a spatial uncertainty but currently producing bugs
|
||||
ordinary_kriging_settings:
|
||||
min_points: 3
|
||||
max_points: 100
|
||||
grid_resolution: 500
|
||||
min_nodes: 10
|
||||
y_min: ~ # manual override for the minimum y value. Leave blank or ~ to use ymin.
|
||||
cut_ground: True # cuts everything below the ground level in the krig (need height_agl to work at the moment)
|
||||
171
src/gasflux/interpolation.py
Normal file
171
src/gasflux/interpolation.py
Normal file
@ -0,0 +1,171 @@
|
||||
"""Functions related to kriging and other kinds of interpolation"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import skgstat as skg
|
||||
from scipy import integrate
|
||||
|
||||
from . import plotting
|
||||
|
||||
|
||||
def simpsonintegrate(array: np.ndarray, x_cell_size: float, y_cell_size: float) -> float:
|
||||
"""Function to obtain the volume of the krig in kgh⁻¹, i.e. the cut-fill volume
|
||||
(negative volumes from background noise are subtracted)."""
|
||||
grid = np.nan_to_num(array.copy(), copy=False, nan=0)
|
||||
vol_rows = integrate.simpson(np.transpose(grid)) # this integrates along each row of the grid
|
||||
vol_grid = integrate.simpson(vol_rows) # this integrates the rows together
|
||||
return vol_grid * x_cell_size * y_cell_size # type: ignore
|
||||
|
||||
|
||||
def directional_gas_semivariogram(
|
||||
df: pd.DataFrame, x: str, z: str, gas: str, semivariogram_filter: float | None = None, **semivariogram_settings
|
||||
):
|
||||
"""Function to calculate the directional semivariogram - typically horizontally - of a gas in a dataframe."""
|
||||
if semivariogram_filter:
|
||||
df = df[df[gas] > semivariogram_filter]
|
||||
v = skg.DirectionalVariogram(
|
||||
df[[x, z]].to_numpy(),
|
||||
df[gas].to_numpy(),
|
||||
**semivariogram_settings,
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
def ordinary_kriging(
|
||||
df: pd.DataFrame,
|
||||
x: str,
|
||||
y: str,
|
||||
gas: str,
|
||||
ordinary_kriging_settings: dict,
|
||||
semivariogram_filter: float | None = None,
|
||||
**semivariogram_settings,
|
||||
):
|
||||
"""Function to calculate the ordinary kriging of a gas in a dataframe, after calculating a semivariogram."""
|
||||
gasflux = f"{gas}_kg_h_m2"
|
||||
skg.plotting.backend("plotly") # type: ignore
|
||||
cut_ground = ordinary_kriging_settings["cut_ground"]
|
||||
semivariogram = directional_gas_semivariogram(df, x, y, gasflux, semivariogram_filter, **semivariogram_settings)
|
||||
ok = skg.OrdinaryKriging(
|
||||
semivariogram,
|
||||
coordinates=df[[x, y]].to_numpy(),
|
||||
values=df[gasflux].to_numpy(),
|
||||
min_points=ordinary_kriging_settings["min_points"],
|
||||
max_points=ordinary_kriging_settings["max_points"],
|
||||
)
|
||||
x_max = df[x].max()
|
||||
x_min = df[x].min()
|
||||
y_max = df[y].max()
|
||||
y_min = df[y].min() if ordinary_kriging_settings["y_min"] is None else ordinary_kriging_settings["y_min"]
|
||||
if cut_ground is True:
|
||||
df["ground_elevation_ato"] = df.loc[:, "height_ato"] - df.loc[:, "height_agl"]
|
||||
y_min = min(df["ground_elevation_ato"].min(), y_min)
|
||||
x_range, y_range = x_max - x_min, y_max - y_min
|
||||
cell_rough_size = np.sqrt((x_range * y_range) / ordinary_kriging_settings["grid_resolution"])
|
||||
x_nodes, y_nodes = (
|
||||
max(int(r / cell_rough_size), ordinary_kriging_settings["min_nodes"]) for r in [x_range, y_range]
|
||||
)
|
||||
x_cell_size = (x_max - x_min) / x_nodes
|
||||
y_cell_size = (y_max - y_min) / y_nodes
|
||||
xx, yy = np.mgrid[
|
||||
x_min : x_max : x_nodes * 1j,
|
||||
y_min : y_max : y_nodes * 1j, # type: ignore
|
||||
] # type: ignore
|
||||
field = ok.transform(xx.flatten(), yy.flatten()).reshape(xx.shape)
|
||||
if cut_ground:
|
||||
field = remove_values_below_ground(df, field, xx, yy)
|
||||
volume = simpsonintegrate(field, x_cell_size, y_cell_size)
|
||||
|
||||
fieldpos = np.copy(field)
|
||||
fieldpos[fieldpos < 0] = 0
|
||||
volumepos = simpsonintegrate(fieldpos, x_cell_size, y_cell_size)
|
||||
|
||||
fieldneg = np.copy(field)
|
||||
fieldneg[fieldneg > 0] = 0
|
||||
volumeneg = simpsonintegrate(fieldneg, x_cell_size, y_cell_size)
|
||||
|
||||
error_1s = ok.sigma.reshape(xx.shape)
|
||||
# np.nan_to_num(error_1s, copy=False, nan=0)
|
||||
volume_error = simpsonintegrate(error_1s, x_cell_size, y_cell_size)
|
||||
|
||||
contour_plot = plotting.contour_krig(df=df, gas=gas, xx=xx, yy=yy, field=field, x=x, y=y, cut_ground=cut_ground)
|
||||
grid_plot = plotting.heatmap_krig(xx, yy, field)
|
||||
output_text = (
|
||||
f"The emissions flux of {gas.upper()} is {volume:.3f}kgh⁻¹; "
|
||||
f"the cut and fill volumes of the grid are {volumepos:.3f} and {volumeneg:.3f}kgh⁻¹. "
|
||||
f"The grid itself is {x_nodes}x{y_nodes} nodes, with nodes measuring {x_cell_size:.2f}m x {y_cell_size:.2f}m."
|
||||
)
|
||||
krig_variables = {
|
||||
"gas": gas,
|
||||
"field": field,
|
||||
"fieldpos": fieldpos,
|
||||
"fieldneg": fieldneg,
|
||||
"xx": xx,
|
||||
"yy": yy,
|
||||
"volume": volume,
|
||||
"volumepos": volumepos,
|
||||
"volumeneg": volumeneg,
|
||||
"error field (1 sigma)": error_1s,
|
||||
"volume_error": volume_error,
|
||||
}
|
||||
semivariogram_plot = semivariogram.plot(show=False)
|
||||
|
||||
return krig_variables, output_text, contour_plot, grid_plot, semivariogram_plot
|
||||
|
||||
|
||||
def remove_values_below_ground(
|
||||
df: pd.DataFrame, field: np.ndarray, xx: np.ndarray, yy: np.ndarray, x: str = "x", alt: str = "height_ato"
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Adjust field values based on elevation data, setting values below ground to NaN.
|
||||
"""
|
||||
max_x = df[x].max()
|
||||
max_y = df[alt].max()
|
||||
|
||||
x_right = np.empty_like(xx)
|
||||
x_right[:-1, :] = xx[1:, :]
|
||||
x_right[-1, :] = max_x
|
||||
x_points = (xx + x_right) / 2
|
||||
|
||||
y_top = np.empty_like(yy)
|
||||
y_top[:, :-1] = yy[:, 1:]
|
||||
y_top[:, -1] = max_y
|
||||
y_points = (yy + y_top) / 2
|
||||
|
||||
x_points_flat = x_points.ravel()
|
||||
y_points_flat = y_points.ravel()
|
||||
|
||||
ground_levels = compute_relative_ground_levels(df, x_points_flat)
|
||||
|
||||
below_ground = y_points_flat < ground_levels
|
||||
field_flat = field.ravel()
|
||||
field_flat[below_ground] = np.nan
|
||||
|
||||
return field_flat.reshape(field.shape)
|
||||
|
||||
|
||||
def compute_relative_ground_levels(
|
||||
df: pd.DataFrame, x_points: np.ndarray, y1: str = "height_agl", y2: str = "height_ato"
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Calculate ground levels at given x coordinates, considering elevation above ground and takeoff altitude. A
|
||||
sort of janky averaged DEM, basically. Will accept either ground elevation and altitude, or height above ground
|
||||
level and height above takeoff as inputs for y1 and y2.
|
||||
"""
|
||||
df_clean = df.dropna(subset=[y1, y2])
|
||||
df_sorted = df_clean.sort_values(by="x").drop_duplicates(subset="x")
|
||||
y1_at_x = np.interp(x_points, df_sorted["x"], df_sorted[y1])
|
||||
y2_at_x = np.interp(x_points, df_sorted["x"], df_sorted[y2])
|
||||
|
||||
return y2_at_x - y1_at_x
|
||||
|
||||
|
||||
# def additive_row_integration(df: pd.DataFrame, rowlabel: str = "slice"):
|
||||
# """2D integration of a dataframe along the x-axis, with the altitude as the y-axis."""
|
||||
# integrals = {}
|
||||
# for i in range(df[rowlabel].max() + 1):
|
||||
# df_slice = df[df[rowlabel] == i]
|
||||
# df_slice = df_slice.sort_values(by="x")
|
||||
# line_integral = integrate.simpson(y=df_slice["ch4_kg_h_m2"], x=df_slice["x"])
|
||||
# area_integral = line_integral * (df_slice["altitude"].max() - df_slice["altitude"].min())
|
||||
# integrals[i] = area_integral
|
||||
# return integrals
|
||||
61
src/gasflux/ml.py
Normal file
61
src/gasflux/ml.py
Normal file
@ -0,0 +1,61 @@
|
||||
"""Experimental module for machine learning flight filtering"""
|
||||
|
||||
import os
|
||||
|
||||
import joblib
|
||||
import pandas as pd
|
||||
|
||||
from . import plotting
|
||||
import plotly.graph_objects as go
|
||||
|
||||
model = None # Lazy loading: Load the model only if it hasn't been loaded yet
|
||||
|
||||
|
||||
def load_model():
|
||||
"""Load the model from the model file path. If the model has already been loaded, return it."""
|
||||
global model
|
||||
if model is None:
|
||||
default_model_path = os.path.join(os.path.dirname(__file__), "resources/model.pkl")
|
||||
model_file_path = os.getenv("GASFLUX_MODEL_PATH", default_model_path)
|
||||
try:
|
||||
model = joblib.load(model_file_path)
|
||||
except FileNotFoundError as e:
|
||||
raise FileNotFoundError(f"Model file not found at {model_file_path}. Please check the file path.") from e
|
||||
except Exception as e:
|
||||
raise Exception("An error occurred while loading the model.") from e
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def make_prediction(
|
||||
df: pd.DataFrame,
|
||||
course_elevation="course_elevation",
|
||||
height_ato="height_ato",
|
||||
horiz_spd="horiz_spd",
|
||||
z_spd="z_spd",
|
||||
) -> tuple[pd.DataFrame, go.Figure]:
|
||||
"""Make predictions based on the input DataFrame and add them to the DataFrame.
|
||||
:param df: DataFrame containing the required features
|
||||
:param course_azimuth: Name of the column containing the course azimuth
|
||||
:param course_elevation: Name of the column containing the course elevation
|
||||
:param height_ato: Name of the column containing the height above take-off
|
||||
:param idx: Name of the column containing the index
|
||||
|
||||
return: Tuple of the DataFrame with the predictions and a Plotly 3D scatter plot of the predictions
|
||||
"""
|
||||
model = load_model()
|
||||
# Ensure the DataFrame contains all the required features
|
||||
required_features = [course_elevation, height_ato, horiz_spd, z_spd]
|
||||
if not all(feature in df.columns for feature in required_features):
|
||||
missing_features = [feature for feature in required_features if feature not in df.columns]
|
||||
raise ValueError(f"DataFrame is missing (or mislabelled) the following required features: {missing_features}")
|
||||
# make idx col if not present
|
||||
if "idx" not in df.columns:
|
||||
df["idx"] = df.index
|
||||
cols_for_model = ["course_elevation", "height_ato", "idx", "horiz_spd", "z_spd"]
|
||||
predictions = model.predict(df[cols_for_model])
|
||||
|
||||
df["predictions"] = predictions
|
||||
fig = plotting.scatter_3d(df)
|
||||
|
||||
return df, fig
|
||||
609
src/gasflux/plotting.py
Normal file
609
src/gasflux/plotting.py
Normal file
@ -0,0 +1,609 @@
|
||||
"""Various plotting functions mainly based around plotly."""
|
||||
|
||||
import matplotlib.colors as mcolors
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import plotly.io as pio
|
||||
import simplekml
|
||||
from plotly.subplots import make_subplots
|
||||
|
||||
from . import processing
|
||||
|
||||
pio.templates["default"] = go.layout.Template(
|
||||
layout=go.Layout(
|
||||
margin=go.layout.Margin(l=0, r=0, b=0, t=0, pad=0),
|
||||
),
|
||||
)
|
||||
|
||||
pio.templates.default = "simple_white+default"
|
||||
|
||||
|
||||
styling = {
|
||||
"colorscale": "geyser",
|
||||
}
|
||||
|
||||
|
||||
def blank_figure():
|
||||
fig = go.Figure()
|
||||
return fig
|
||||
|
||||
|
||||
def scatter_3d(
|
||||
df: pd.DataFrame,
|
||||
color: str = "",
|
||||
colorbar_title: str = "",
|
||||
timestamp: str = "timestamp",
|
||||
x: str = "utm_easting",
|
||||
y: str = "utm_northing",
|
||||
z: str = "height_ato",
|
||||
courses: bool = False,
|
||||
):
|
||||
fig = px.scatter_3d(df, x=x, y=y, z=z)
|
||||
|
||||
if color:
|
||||
custom_data = [df[timestamp]]
|
||||
if courses:
|
||||
custom_data.extend([df["course_elevation"], df["course_azimuth"]])
|
||||
custom_data = np.stack(custom_data, axis=-1)
|
||||
hover_template = [
|
||||
f"{x}: %{{x:.2f}}",
|
||||
f"{y}: %{{y:.2f}}",
|
||||
f"{z}: %{{z:.2f}}",
|
||||
f"{color}: %{{marker.color:.2f}}",
|
||||
f"{timestamp}: %{{customdata[0]|%Y-%m-%d %H:%M:%S}}",
|
||||
"Index: %{pointNumber}",
|
||||
]
|
||||
|
||||
if courses:
|
||||
hover_template.extend(
|
||||
[
|
||||
"Course Elevation: %{customdata[1]:.2f}",
|
||||
"Course Azimuth: %{customdata[2]:.2f}",
|
||||
]
|
||||
)
|
||||
hover_template_str = "<br>".join(hover_template)
|
||||
fig.update_traces(
|
||||
marker=dict(
|
||||
color=df[color],
|
||||
size=4,
|
||||
opacity=0.5,
|
||||
colorscale=styling["colorscale"],
|
||||
colorbar=dict(title=colorbar_title),
|
||||
),
|
||||
customdata=custom_data,
|
||||
hovertemplate=hover_template_str,
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def scatter_2d(
|
||||
df: pd.DataFrame,
|
||||
x: str,
|
||||
color: str,
|
||||
y: str = "height_ato",
|
||||
**kwargs,
|
||||
):
|
||||
fig = px.scatter(
|
||||
df,
|
||||
x=x,
|
||||
y=y,
|
||||
color=color,
|
||||
color_continuous_scale=styling["colorscale"],
|
||||
opacity=0.8,
|
||||
**kwargs,
|
||||
)
|
||||
fig.update_traces(
|
||||
customdata=df.index,
|
||||
hovertemplate="<br>".join(
|
||||
[
|
||||
"x: %{x:.2f}",
|
||||
"height_ato: %{y:.2f}",
|
||||
f"{color}: %{{marker.color:.2f}}",
|
||||
"Time: %{customdata}",
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def time_series(
|
||||
df: pd.DataFrame,
|
||||
ys: str | list[str],
|
||||
x: str = "timestamp",
|
||||
color: str | None = None,
|
||||
split=None,
|
||||
y_mins: float | list[float | int] | None = None,
|
||||
rolling_average: bool = True,
|
||||
scatter: bool = True,
|
||||
rolling_window: int = 5,
|
||||
y_titles: str | list[str] | None = None,
|
||||
legend: bool = True,
|
||||
) -> go.Figure:
|
||||
colors = px.colors.qualitative.Plotly
|
||||
|
||||
if isinstance(ys, str):
|
||||
ys = [ys]
|
||||
if y_titles is None:
|
||||
y_titles = ys
|
||||
single_title = False
|
||||
elif isinstance(y_titles, str):
|
||||
y_titles = [y_titles]
|
||||
single_title = True
|
||||
elif isinstance(y_titles, list):
|
||||
if len(y_titles) != len(ys):
|
||||
raise ValueError("Length of y_titles must be equal to length of ys")
|
||||
single_title = False
|
||||
else:
|
||||
raise ValueError("Invalid y_titles value")
|
||||
if isinstance(y_mins, (float | int)):
|
||||
y_mins = [y_mins]
|
||||
if isinstance(y_mins, list):
|
||||
if len(y_mins) != len(ys):
|
||||
raise ValueError("Length of y_mins must be equal to length of ys")
|
||||
|
||||
fig = go.Figure()
|
||||
|
||||
axis_space = 0.05
|
||||
domain_start = axis_space * (len(ys)) if len(ys) > 1 else 0
|
||||
fig.update_layout(
|
||||
xaxis=dict(
|
||||
domain=[domain_start, 1],
|
||||
),
|
||||
)
|
||||
|
||||
for i, y in enumerate(ys):
|
||||
yaxis_name = f"yaxis{i+1}"
|
||||
yaxis_ref = f"y{i+1}"
|
||||
|
||||
trace_color = "black" if single_title and i == 0 else colors[i % len(colors)]
|
||||
|
||||
marker_i = dict(size=8, opacity=0.3 if rolling_average else 0.5, color=trace_color)
|
||||
if color is not None:
|
||||
marker_i["color"] = df[color] # type: ignore
|
||||
marker_i["colorscale"] = styling["colorscale"]
|
||||
|
||||
hover_template = f"{x}: %{{x}}<br>{y}: %{{y:.2f}}<br>"
|
||||
if color:
|
||||
hover_template += f"{color}: %{{marker.color:.2f}}<br>"
|
||||
|
||||
if scatter:
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[y],
|
||||
name=y,
|
||||
mode="markers",
|
||||
marker=marker_i,
|
||||
yaxis=yaxis_ref,
|
||||
hovertemplate=hover_template,
|
||||
showlegend=legend,
|
||||
)
|
||||
)
|
||||
|
||||
if rolling_average:
|
||||
df[f"rolling_avg_{i}"] = df[y].rolling(window=rolling_window, min_periods=1).mean()
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[f"rolling_avg_{i}"],
|
||||
name=f"{y} {rolling_window}-point avg",
|
||||
mode="lines",
|
||||
line=dict(color=trace_color, width=2),
|
||||
yaxis=yaxis_ref,
|
||||
showlegend=legend,
|
||||
)
|
||||
)
|
||||
|
||||
y_data = df[y]
|
||||
y_min_var = y_data.min()
|
||||
y_max_var = y_data.max()
|
||||
y_range = y_max_var - y_min_var or y_max_var * 0.05
|
||||
|
||||
y_axis_min = y_mins[i] if y_mins is not None and y_mins[i] is not None else y_min_var - y_range * 0.05
|
||||
y_axis_max = y_max_var + y_range * 0.05
|
||||
|
||||
if single_title and i == 0:
|
||||
axis_title = dict(text=y_titles[0], font=dict(color="black"))
|
||||
elif not single_title:
|
||||
axis_title = dict(text=y_titles[i], font=dict(color=trace_color))
|
||||
else:
|
||||
axis_title = None
|
||||
|
||||
axis_config = dict(
|
||||
title=axis_title,
|
||||
tickfont=dict(color=trace_color),
|
||||
range=[y_axis_min, y_axis_max],
|
||||
side="left",
|
||||
position=axis_space * i if i > 0 else None,
|
||||
anchor="free" if i > 0 else None,
|
||||
overlaying="y" if i > 0 else None,
|
||||
showgrid=(i == 0),
|
||||
)
|
||||
|
||||
fig.layout[yaxis_name] = axis_config
|
||||
|
||||
if split is not None:
|
||||
fig.add_shape(
|
||||
type="line",
|
||||
xref="x",
|
||||
yref="paper",
|
||||
x0=split,
|
||||
y0=0,
|
||||
x1=split,
|
||||
y1=1,
|
||||
line=dict(color="red", width=2),
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def background_plotting(df: pd.DataFrame, gas: str):
|
||||
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
||||
ymin = df[gas].min()
|
||||
ymax = df[gas].max()
|
||||
ylim = [ymin * 0.95, ymax * 1.05]
|
||||
y2min = df[f"{gas}_normalised"].min()
|
||||
y2lim = (y2min, y2min + (ylim[1] - ylim[0]))
|
||||
fig.update_yaxes(range=ylim, secondary_y=False, title_text=f"Sensor {gas} (ppm)")
|
||||
fig.update_yaxes(range=y2lim, secondary_y=True, title_text=f"Normalised {gas} (ppm)")
|
||||
fig.add_scatter(x=df["timestamp"], y=df[gas], opacity=0.3, name="Raw Data")
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"], y=df[f"{gas}_fit"], mode="lines", name="Fitted Background", line=dict(dash="dash")
|
||||
)
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"], y=df[f"{gas}_normalised"], yaxis="y2", name="Normalised Data", mode="lines", opacity=0.5
|
||||
)
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"],
|
||||
y=np.where(df[f"{gas}_signal"], df[f"{gas}_normalised"], np.nan),
|
||||
yaxis="y2",
|
||||
name="Classed as signal",
|
||||
mode="lines",
|
||||
opacity=0.5,
|
||||
# color
|
||||
# mode="markers",
|
||||
# marker=dict(size=3),
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def windrose_process(df: pd.DataFrame):
|
||||
beaufort = {
|
||||
"0": [0, 1],
|
||||
"1": [1, 2],
|
||||
"2": [2, 4],
|
||||
"3": [4, 6],
|
||||
"4": [6, 9],
|
||||
"5": [9, 11],
|
||||
"6": [11, 14],
|
||||
"7": [14, 17],
|
||||
"8": [17, 21],
|
||||
"9": [21, 25],
|
||||
"10": [25, 29],
|
||||
"11": [29, 33],
|
||||
"12": [33, 200],
|
||||
}
|
||||
|
||||
beaufort_ms = {
|
||||
"0": "0-1",
|
||||
"1": "1-2",
|
||||
"2": "2-4",
|
||||
"3": "4-6",
|
||||
"4": "6-9",
|
||||
"5": "9-11",
|
||||
"6": "11-14",
|
||||
"7": "14-17",
|
||||
"8": "17-21",
|
||||
"9": "21-25",
|
||||
"10": "25-29",
|
||||
"11": "29-33",
|
||||
"12": "33+",
|
||||
}
|
||||
|
||||
cardinals = {
|
||||
"N1": [0, 11.25],
|
||||
"NNE": [11.25, 33.75],
|
||||
"NE": [33.75, 56.25],
|
||||
"ENE": [56.25, 78.75],
|
||||
"E": [78.75, 101.25],
|
||||
"ESE": [101.25, 123.75],
|
||||
"SE": [123.75, 146.25],
|
||||
"SSE": [146.25, 168.75],
|
||||
"S": [168.75, 191.25],
|
||||
"SSW": [191.25, 213.75],
|
||||
"SW": [213.75, 236.25],
|
||||
"WSW": [236.25, 258.75],
|
||||
"W": [258.75, 281.25],
|
||||
"WNW": [281.25, 303.75],
|
||||
"NW": [303.75, 326.25],
|
||||
"NNW": [326.25, 348.75],
|
||||
"N2": [348.75, 360],
|
||||
}
|
||||
df["wind_direction_bin"] = pd.cut(
|
||||
df["winddir"],
|
||||
bins=[lower for lower, upper in cardinals.values()] + [list(cardinals.values())[-1][1]],
|
||||
labels=[key for key in cardinals],
|
||||
right=False,
|
||||
)
|
||||
|
||||
df["wind_direction_bin"] = (
|
||||
df["wind_direction_bin"].map(lambda x: "N" if x in ["N1", "N2"] else x).astype("category")
|
||||
)
|
||||
df["beaufort"] = pd.cut(
|
||||
df["windspeed"],
|
||||
bins=[lower for lower, upper in beaufort.values()] + [list(beaufort.values())[-1][1]],
|
||||
labels=[key for key in beaufort],
|
||||
right=False,
|
||||
)
|
||||
df["beaufort_ms"] = df["beaufort"].map(beaufort_ms)
|
||||
df_windrose = df.groupby(["wind_direction_bin", "beaufort"], observed=False).size().reset_index(name="count") # type: ignore
|
||||
df_windrose["frequency"] = df_windrose["count"] / df_windrose["count"].sum() * 100
|
||||
df_windrose["wind_direction_bin_degs"] = df_windrose["wind_direction_bin"].cat.rename_categories(
|
||||
{
|
||||
"N": 0,
|
||||
"NNE": 22.5,
|
||||
"NE": 45,
|
||||
"ENE": 67.5,
|
||||
"E": 90,
|
||||
"ESE": 112.5,
|
||||
"SE": 135,
|
||||
"SSE": 157.5,
|
||||
"S": 180,
|
||||
"SSW": 202.5,
|
||||
"SW": 225,
|
||||
"WSW": 247.5,
|
||||
"W": 270,
|
||||
"WNW": 292.5,
|
||||
"NW": 315,
|
||||
"NNW": 337.5,
|
||||
},
|
||||
)
|
||||
df_windrose["beaufort"] = df_windrose["beaufort"].astype(int)
|
||||
return df_windrose
|
||||
|
||||
|
||||
def windrose_graph(df, plot_transect=False, theta1=None, theta2=None):
|
||||
n_colors = 13
|
||||
colors = px.colors.sample_colorscale("turbo", [n / (n_colors - 1) for n in range(n_colors)])
|
||||
fig = px.bar_polar(
|
||||
df,
|
||||
r="frequency",
|
||||
theta="wind_direction_bin_degs",
|
||||
color="beaufort",
|
||||
labels={
|
||||
"frequency": "Frequency (%)",
|
||||
"wind_direction_bin": "Direction",
|
||||
"beaufort": "Beaufort Scale",
|
||||
},
|
||||
color_discrete_map=colors,
|
||||
)
|
||||
fig.update_layout(polar=dict(radialaxis={"visible": False, "showticklabels": False}))
|
||||
fig.update_layout(
|
||||
polar=dict(
|
||||
angularaxis={
|
||||
"showgrid": False,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
fig.update_layout(polar_bargap=0)
|
||||
if plot_transect:
|
||||
max_freq = df.groupby("wind_direction_bin", observed=False)["frequency"].sum().max()
|
||||
fig.add_trace(
|
||||
go.Scatterpolar(
|
||||
r=[max_freq, max_freq],
|
||||
theta=[theta1, theta2],
|
||||
mode="lines",
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
showlegend=False,
|
||||
),
|
||||
)
|
||||
return fig
|
||||
|
||||
|
||||
def windrose(df: pd.DataFrame, plot_transect=False):
|
||||
df_windrose = windrose_process(df)
|
||||
if plot_transect:
|
||||
theta1, theta2 = processing.bimodal_azimuth(df)
|
||||
fig = windrose_graph(df_windrose, plot_transect=plot_transect, theta1=theta1, theta2=theta2)
|
||||
else:
|
||||
fig = windrose_graph(df_windrose, plot_transect=plot_transect)
|
||||
return fig
|
||||
|
||||
|
||||
def outliers(original_data: pd.Series, fence_high: float, fence_low: float):
|
||||
outliers = np.array(original_data > fence_high) | (original_data < fence_low)
|
||||
|
||||
fig = make_subplots(rows=1, cols=2, shared_yaxes=True)
|
||||
fig.add_trace(px.strip(original_data, color=outliers).data[0], row=1, col=1)
|
||||
if sum(outliers) > 0:
|
||||
fig.add_trace(px.strip(original_data, color=outliers).data[1], row=1, col=1)
|
||||
fig.add_shape(
|
||||
go.layout.Shape(
|
||||
type="line",
|
||||
x0=-0.5,
|
||||
y0=fence_high,
|
||||
x1=0.5,
|
||||
y1=fence_high,
|
||||
line=dict(color="red", width=2),
|
||||
),
|
||||
row=1,
|
||||
col=1,
|
||||
)
|
||||
fig.add_shape(
|
||||
go.layout.Shape(
|
||||
type="line",
|
||||
x0=-0.5,
|
||||
y0=fence_low,
|
||||
x1=0.5,
|
||||
y1=fence_low,
|
||||
line=dict(color="red", width=2),
|
||||
),
|
||||
row=1,
|
||||
col=1,
|
||||
)
|
||||
fig.update_traces(offsetgroup=0)
|
||||
|
||||
fig.add_trace(px.scatter(original_data, color=outliers).data[0], row=1, col=2)
|
||||
if sum(outliers) > 0:
|
||||
fig.add_trace(px.scatter(original_data, color=outliers).data[1], row=1, col=2)
|
||||
fig.update_layout(showlegend=False, yaxis_title="Windspeed (ms⁻¹)")
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def contour_krig(
|
||||
df: pd.DataFrame,
|
||||
gas: str,
|
||||
# array of float 64
|
||||
xx: np.ndarray,
|
||||
yy: np.ndarray,
|
||||
field: np.ndarray,
|
||||
cut_ground: bool = False,
|
||||
x: str = "x",
|
||||
y: str = "height_ato",
|
||||
) -> go.Figure:
|
||||
if np.isnan(field).all():
|
||||
return blank_figure()
|
||||
fig = go.Figure()
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[y],
|
||||
mode="markers",
|
||||
marker={
|
||||
"color": df[f"{gas}_normalised"],
|
||||
"colorscale": styling["colorscale"],
|
||||
"showscale": True,
|
||||
"colorbar": {
|
||||
"title": f"{gas} (ppm)",
|
||||
},
|
||||
},
|
||||
showlegend=False,
|
||||
)
|
||||
)
|
||||
fig.add_trace(
|
||||
go.Contour(
|
||||
z=field.T,
|
||||
x=xx[:, 0],
|
||||
y=yy[0, :],
|
||||
contours={
|
||||
"start": field.min(),
|
||||
"end": field.max(),
|
||||
"size": (field[~np.isnan(field)].max() - field[~np.isnan(field)].min()) / 21,
|
||||
},
|
||||
colorscale=styling["colorscale"],
|
||||
opacity=0.5,
|
||||
showlegend=False,
|
||||
showscale=False,
|
||||
)
|
||||
)
|
||||
fig.update_xaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="horizontal distance on projected flux plane (m)",
|
||||
range=[np.min(xx), np.max(xx)],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=20,
|
||||
)
|
||||
fig.update_yaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="height above takeoff (m)",
|
||||
range=[np.min(yy), np.max(yy)],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=10,
|
||||
)
|
||||
if cut_ground:
|
||||
resolution = 200 # how many points to interpolate over
|
||||
df["ground_elevation_ato"] = df.loc[:, "height_ato"] - df.loc[:, "height_agl"]
|
||||
df_sorted = df.dropna(subset=[x, "ground_elevation_ato"]).sort_values(x)
|
||||
x_min, x_max = df_sorted[x].min(), df_sorted[x].max()
|
||||
x_interp = np.linspace(x_min, x_max, resolution)
|
||||
ground_ato_interp = np.interp(x_interp, df_sorted[x], df_sorted["ground_elevation_ato"])
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=x_interp,
|
||||
y=ground_ato_interp,
|
||||
mode="lines",
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
name="Interpolated Ground Level",
|
||||
)
|
||||
)
|
||||
fig.layout.coloraxis.colorbar.title = "Emissions flux (kg⋅m⁻²⋅h⁻¹)"
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def heatmap_krig(xx: np.ndarray, yy: np.ndarray, field: np.ndarray):
|
||||
fig = px.imshow(field.T, x=xx[:, 0], y=yy[0, :], color_continuous_scale=styling["colorscale"], origin="lower")
|
||||
fig.layout.coloraxis.colorbar.title = "Emissions flux (kg⋅m⁻²⋅h⁻¹)"
|
||||
fig.update_xaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="horizontal distance on cylindrical projected flux plane (m)",
|
||||
range=[xx.min(), xx.max()],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=20,
|
||||
)
|
||||
fig.update_yaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="height above ground level (m)",
|
||||
range=[yy.min(), yy.max()],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=10,
|
||||
)
|
||||
fig.update_layout(coloraxis_colorbar=dict(len=0.25))
|
||||
return fig
|
||||
|
||||
|
||||
def create_kml_file(data: pd.DataFrame, output_file: str, column: str, altitudemode: str):
|
||||
kml = simplekml.Kml()
|
||||
|
||||
min_value = data[column].min()
|
||||
max_value = data[column].max()
|
||||
|
||||
custom_colors = [
|
||||
"#008080",
|
||||
"#70a494",
|
||||
"#b4c8a8",
|
||||
"#f6edbd",
|
||||
"#edbb8a",
|
||||
"#de8a5a",
|
||||
"#ca562c",
|
||||
] # based on plotly geyser
|
||||
cmap = mcolors.LinearSegmentedColormap.from_list("custom_cmap", custom_colors)
|
||||
|
||||
for _index, row in data.iterrows():
|
||||
col_normalized = (row[column] - min) / (max_value - min_value)
|
||||
color = mcolors.rgb2hex(cmap(col_normalized))
|
||||
|
||||
pnt = kml.newpoint(coords=[(row["longitude"], row["latitude"], row["height_ato"])], altitudemode=altitudemode)
|
||||
pnt.iconstyle.icon.href = "http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png"
|
||||
pnt.iconstyle.color = simplekml.Color.rgb(int(color[1:3], 16), int(color[3:5], 16), int(color[5:], 16))
|
||||
pnt.iconstyle.scale = 0.6
|
||||
pnt.description = f"Concentration: {row[column]} ppm"
|
||||
|
||||
kml.save(output_file)
|
||||
109
src/gasflux/pre_processing.py
Normal file
109
src/gasflux/pre_processing.py
Normal file
@ -0,0 +1,109 @@
|
||||
"""Functions that organise the data into standard columns in pandas dataframes. Conversion functions (e.g. WGS84 to UTM)
|
||||
are here but transformations take place in processing.py"""
|
||||
|
||||
import geopandas as gpd
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from . import plotting
|
||||
from .processing import circ_median
|
||||
|
||||
|
||||
def data_tests(df: pd.DataFrame):
|
||||
assert df["ch4"].min() > 1.6, "ch4 values are too low"
|
||||
assert df.index.is_monotonic_increasing, "data is not sorted by time"
|
||||
assert df.index.is_unique, "data has duplicate timestamps"
|
||||
assert df["ch4"].isna().sum() == 0, "ch4 has missing values"
|
||||
assert df["windspeed"].min() >= 0, "windspeed values are negative"
|
||||
assert df["windspeed"].max() < 20, "windspeed values are too high"
|
||||
if df["windspeed"].max() > 15:
|
||||
print("Warning: windspeed is greater than 15 m/s, perhaps due to errors in the data.")
|
||||
|
||||
|
||||
# make timestamp column from UTCs, Month, Day, Year
|
||||
def timestamp_from_four_columns(df):
|
||||
df["Year"] = df["Year"] + 2000
|
||||
df["time"] = pd.to_datetime(df["UTCs"], unit="s")
|
||||
df["date"] = pd.to_datetime(df[["Year", "Month", "Day"]])
|
||||
df["timestamp"] = pd.to_datetime(df["date"].dt.date.astype(str) + " " + df["time"].dt.time.astype(str))
|
||||
df.index = df["timestamp"]
|
||||
df.drop(
|
||||
["Year", "Month", "Day", "time", "date", "timestamp", "UTCs"],
|
||||
axis=1,
|
||||
inplace=True,
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
# add UTM from latitudes and longitudes
|
||||
def add_utm(df: pd.DataFrame) -> pd.DataFrame:
|
||||
gdf = gpd.GeoDataFrame( # type: ignore
|
||||
df,
|
||||
geometry=gpd.points_from_xy(df["longitude"], df["latitude"], crs="EPSG:4326"),
|
||||
)
|
||||
utm = gdf.estimate_utm_crs()
|
||||
gdf = gdf.to_crs(utm)
|
||||
if not isinstance(gdf, gpd.GeoDataFrame):
|
||||
raise TypeError("Failed to reproject to a GeoDataFrame")
|
||||
gdf["utm_easting"] = gdf.geometry.x
|
||||
gdf["utm_northing"] = gdf.geometry.y
|
||||
output_df = pd.DataFrame(gdf.drop(columns="geometry"))
|
||||
|
||||
return output_df
|
||||
|
||||
|
||||
# add columns for drone course azimuth and elevation
|
||||
def add_course(df, rolling_window=1):
|
||||
df["hor_distance"] = np.sqrt((df["utm_northing"].diff()) ** 2 + (df["utm_easting"].diff()) ** 2)
|
||||
df["vert_distance"] = df["height_ato"].diff()
|
||||
df["vert_distance"] = pd.to_numeric(df["vert_distance"], errors="coerce")
|
||||
df["hor_distance"] = pd.to_numeric(df["hor_distance"], errors="coerce")
|
||||
df["course_azimuth"] = (
|
||||
(np.degrees(np.arctan2(df["utm_easting"].diff(), df["utm_northing"].diff())) % 360)
|
||||
.rolling(rolling_window)
|
||||
.apply(lambda x: circ_median(x), raw=True)
|
||||
)
|
||||
df["course_elevation"] = (
|
||||
np.degrees(np.arctan2(df["vert_distance"], df["hor_distance"]))
|
||||
.rolling(rolling_window)
|
||||
.apply(lambda x: circ_median(x), raw=True)
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def manual_filtering(dict_dfs: dict, split_times: dict, mask_spans: dict) -> dict:
|
||||
filtered_dfs = {}
|
||||
for name, df in dict_dfs.items():
|
||||
if name in mask_spans:
|
||||
for i in range(len(mask_spans[name])):
|
||||
df = df.drop(
|
||||
df.between_time(mask_spans[name][i].split(" - ")[0], mask_spans[name][i].split(" - ")[1]).index,
|
||||
).copy()
|
||||
filtered_dfs[name] = df.copy()
|
||||
if name in split_times:
|
||||
split_times[name].append("23:59:59")
|
||||
split_times[name].insert(0, "00:00:00")
|
||||
for i in range(len(split_times[name]) - 1):
|
||||
df2 = df.between_time(split_times[name][i], split_times[name][i + 1]).copy()
|
||||
filtered_dfs[name + "_" + str(i)] = df2.copy()
|
||||
elif name not in split_times:
|
||||
filtered_dfs[name] = df.copy()
|
||||
return filtered_dfs
|
||||
|
||||
|
||||
def remove_outliers(df: pd.DataFrame, column: str, name: str):
|
||||
q1 = df[column].quantile(0.25)
|
||||
q3 = df[column].quantile(0.75)
|
||||
iqr = q3 - q1
|
||||
fence_low = q1 - 3 * iqr
|
||||
fence_high = q3 + 3 * iqr
|
||||
fig = plotting.outliers(df[column], fence_high, fence_low)
|
||||
outliers = df.loc[(df[column] < fence_low) | (df[column] > fence_high)]
|
||||
if len(outliers) > 0:
|
||||
print(f"{len(outliers)} outliers removed from {name} {column} data")
|
||||
# nan for outliers, not row removal
|
||||
df.loc[(df[column] < fence_low) | (df[column] > fence_high), column] = float("nan")
|
||||
elif len(outliers) == 0:
|
||||
print(f"No outliers found in {name} {column} data")
|
||||
|
||||
return df, fig
|
||||
650
src/gasflux/processing.py
Normal file
650
src/gasflux/processing.py
Normal file
@ -0,0 +1,650 @@
|
||||
"""Processing function, usually implying some kind of filtering or data transformation."""
|
||||
|
||||
from matplotlib.figure import Figure
|
||||
from itertools import groupby
|
||||
from typing import Any
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy import odr
|
||||
from scipy.optimize import least_squares
|
||||
from scipy.signal import find_peaks
|
||||
from scipy.stats import circmean
|
||||
import warnings
|
||||
import logging
|
||||
|
||||
|
||||
def circ_median(x: np.ndarray | pd.Series) -> float:
|
||||
"""
|
||||
Finds the largest "empty" arc in the circle, splits it there and returns the median of the straight line.
|
||||
Probably there are more elegant ways... see https://github.com/scipy/scipy/issues/6644
|
||||
|
||||
Parameters:
|
||||
x (np.ndarray | pd.Series): The input array of angles.
|
||||
|
||||
Returns:
|
||||
float: The circular median of the input angles.
|
||||
"""
|
||||
angles = sorted(x)
|
||||
n = len(angles)
|
||||
if n == 1:
|
||||
return angles[0]
|
||||
gaps = [(angles[(i + 1) % n] - angles[i]) % 360 for i in range(n)]
|
||||
largest_gap_index = gaps.index(max(gaps))
|
||||
rearranged_angles = angles[largest_gap_index + 1 :] + angles[: largest_gap_index + 1]
|
||||
if n % 2 == 1:
|
||||
return rearranged_angles[n // 2]
|
||||
else:
|
||||
return (rearranged_angles[n // 2 - 1] + rearranged_angles[n // 2]) % 360 / 2
|
||||
|
||||
|
||||
def min_angular_displacement(x: float | np.ndarray, y: float | np.ndarray) -> float | np.ndarray:
|
||||
"""
|
||||
Calculates the minimum circular difference between two angles (in 360 degree space)
|
||||
"""
|
||||
return np.minimum(np.abs(x - y) % 360, (360 - np.abs(x - y)) % 360)
|
||||
|
||||
|
||||
def wind_offset_correction(df: pd.DataFrame, plane_angle: float) -> pd.DataFrame:
|
||||
"""
|
||||
Corrects wind direction data for a given plane angle, assuming the plane is the primary orientation of the dataset.
|
||||
This is useful for aligning wind data with the plane's orientation, facilitating analysis of wind effects.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe containing wind direction data.
|
||||
wind_dir_col (str): Column name for wind direction data.
|
||||
plane_angle (float): Angle of the plane in degrees.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: The modified dataframe with corrected wind direction data.
|
||||
"""
|
||||
df = df.copy()
|
||||
df["winddir_rel"] = df.apply(lambda row: abs(90 - min_angular_displacement(row["winddir"], plane_angle)), axis=1) # type: ignore
|
||||
df["windspeed_measured"] = df["windspeed"]
|
||||
df["windspeed"] = df["windspeed"] * np.cos(np.radians(df["winddir_rel"]))
|
||||
return df
|
||||
|
||||
|
||||
def bimodal_azimuth(
|
||||
df: pd.DataFrame, course_col: str = "course_azimuth", min_height: int = 5, min_diff: int = 160
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Identifies the two most frequent course azimuths in the dataset, ensuring they are sufficiently
|
||||
distinct. Filters data by height and removes NaNs before analysis.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe.
|
||||
course_col (str): Column name for course azimuth data. Default is "course_azimuth".
|
||||
min_altitude (int): Minimum height (_ato typically) for data to be included. Default is 5.
|
||||
min_diff (int): Minimum difference between the two modes. Default is 160.
|
||||
|
||||
Returns:
|
||||
tuple: Two modes of the course azimuth.
|
||||
"""
|
||||
df = df[df["height_ato"] >= min_height]
|
||||
data = df[course_col].dropna().to_numpy()
|
||||
hist, edges = np.histogram(data, bins=50)
|
||||
edgedist = edges[1] - edges[0]
|
||||
bin_centers = edges[:-1] + edgedist / 2
|
||||
max_freq_idx = np.argsort(hist)[-2:] # top 2 frequencies
|
||||
mode1, mode2 = bin_centers[max_freq_idx]
|
||||
while min_angular_displacement(mode1, mode2) < min_diff:
|
||||
if hist[max_freq_idx[0]] < hist[max_freq_idx[1]]:
|
||||
hist[max_freq_idx[0]] = 0
|
||||
max_freq_idx = np.argsort(hist)[-2:]
|
||||
else:
|
||||
hist[max_freq_idx[1]] = 0
|
||||
max_freq_idx = np.argsort(hist)[-2:]
|
||||
mode1, mode2 = bin_centers[max_freq_idx]
|
||||
if min_angular_displacement(mode1, mode2) < 160:
|
||||
warnings.warn(
|
||||
f"Two modes are close together - this probably should never happen: {mode1:.2f} and {mode2:.2f}",
|
||||
UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return (mode1, mode2)
|
||||
|
||||
|
||||
# this returns modes of slope from -90 to 90 degrees.
|
||||
def bimodal_elevation(
|
||||
df: pd.DataFrame, course_col: str = "course_elevation", min_height: float = 5, max_slope: float = 70
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Identifies the most frequent course elevation in the dataset, adjusted for vertical movements.
|
||||
Filters data by height and removes NaNs before analysis.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe.
|
||||
course_col (str): Column name for course elevation data. Default is "course_elevation".
|
||||
min_height (int): Minimum height for data to be included. Default is 5.
|
||||
max_slope (int): Maximum slope to consider, avoTupleiding vertical movements. Default is 70.
|
||||
|
||||
Returns:
|
||||
tuple: Mode of course elevation and its negative, representing possible ascent/descent angles.
|
||||
"""
|
||||
df = df[df["height_ato"] >= df["height_ato"].min() + min_height]
|
||||
data = df[course_col].to_numpy()
|
||||
data = np.abs(data[~np.isnan(data)])
|
||||
# to get around the edge case where vertical movements are modal
|
||||
data = data[data < max_slope]
|
||||
hist, edges = np.histogram(data, bins=50)
|
||||
max_freq_idx = np.argsort(hist)[::-1][:2]
|
||||
mode = edges[max_freq_idx][0]
|
||||
return (mode, -mode)
|
||||
|
||||
|
||||
def height_transect_splitter(df: pd.DataFrame, height_col: str = "height_ato") -> tuple[pd.DataFrame, Figure]:
|
||||
"""
|
||||
Splits the dataset into height-based transects and plots histogram peaks to identify prominent
|
||||
height ranges. Only works if the flights are flat.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe containing height data.
|
||||
|
||||
Returns:
|
||||
tuple: Modified dataframe with transect labels and a figure showing the histogram with peaks.
|
||||
"""
|
||||
df = df.copy()
|
||||
heights = df[height_col].to_numpy()
|
||||
counts, bin_edges = np.histogram(heights, bins=40)
|
||||
bin_width = bin_edges[1] - bin_edges[0]
|
||||
bin_edges = np.append(heights.min() - bin_width, bin_edges) # avoid literal edge effects
|
||||
bin_edges = np.append(bin_edges, heights.max() + bin_width)
|
||||
counts = np.append(0, counts)
|
||||
counts = np.append(counts, 0)
|
||||
bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
|
||||
peaks, properties = find_peaks(counts)
|
||||
transect_edges = (bin_centers[peaks][:-1] + bin_centers[peaks][1:]) / 2
|
||||
transect_edges = np.append(heights.min(), transect_edges)
|
||||
transect_edges = np.append(transect_edges, heights.max())
|
||||
fig, ax = plt.subplots()
|
||||
ax.stairs(edges=bin_edges, values=counts, fill=True)
|
||||
ax.plot(bin_centers[peaks], counts[peaks], "x", color="red")
|
||||
ax.vlines(transect_edges, ymin=0, ymax=max(counts), color="red")
|
||||
df["transect_num"] = pd.cut(df[height_col], bins=list(transect_edges), labels=False, include_lowest=True) # type: ignore
|
||||
return df, fig
|
||||
|
||||
|
||||
def add_transect_azimuth_switches(df: pd.DataFrame, threshold=150, shift=3) -> pd.DataFrame:
|
||||
"""
|
||||
Identifies transects based on significant changes in course azimuth, incrementing a transect
|
||||
counter to distinguish different flight paths. This is a really crude function and should probably
|
||||
be only used with data that's already been filtered in some way.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe with course azimuths.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: The modified dataframe with a new 'transect' column indicating transect IDs.
|
||||
"""
|
||||
df = df.copy()
|
||||
df["transect_num"] = 0
|
||||
df["prev_course_azimuth"] = df["course_azimuth"].shift(shift) # this gives better behaviour for very neat transects
|
||||
df["deg_displace"] = df.apply(
|
||||
lambda row: min_angular_displacement(row["course_azimuth"], row["prev_course_azimuth"]) # type: ignore
|
||||
if not pd.isnull(row["prev_course_azimuth"])
|
||||
else np.nan,
|
||||
axis=1,
|
||||
)
|
||||
df.loc[df["deg_displace"] > threshold, "transect_num"] = 1
|
||||
# remove ones that are next to each other
|
||||
for i in range(1, len(df)):
|
||||
if (df.loc[i, "transect_num"] == 1 and df.loc[i - 1, "transect_num"] == 1) or (
|
||||
df.loc[i, "transect_num"] == 1 and df.loc[i - 2, "transect_num"] == 1
|
||||
):
|
||||
df.loc[i, "transect_num"] = 0
|
||||
df["transect_num"] = df["transect_num"].shift(-shift) # recorrect for shift
|
||||
df["transect_num"] = df["transect_num"].cumsum() + 1 # 1 indexed
|
||||
df["transect_num"] = df["transect_num"].ffill()
|
||||
df = df.drop(columns=["prev_course_azimuth", "deg_displace"])
|
||||
return df
|
||||
|
||||
|
||||
def course_filter(
|
||||
df: pd.DataFrame, azimuth_filter: float, azimuth_window: int, elevation_filter: float
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""
|
||||
Filters data based on specified course azimuth and elevation, aiming to isolate transects that
|
||||
align with main flight directions. First elevation is filtered to remove significant climbs or descents
|
||||
(beware terrain-following flights). Bimodal course azimuths are calculated and used to filter the
|
||||
data based on the main flight directions, with a rolling median applied (the window) to smooth the data.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe.
|
||||
azimuth_filter (float): The tolerance for deviation from the main course azimuths.
|
||||
azimuth_window (int): The window size for rolling median calculation of course azimuths.
|
||||
elevation_filter (float): The tolerance for deviation from horizontal flight.
|
||||
|
||||
Returns:
|
||||
tuple: The filtered dataframe and the original unfiltered dataframe for comparison.
|
||||
"""
|
||||
df_filtered = df.copy()
|
||||
df_filtered = df_filtered[abs(df["course_elevation"]) < elevation_filter]
|
||||
|
||||
azi1, azi2 = bimodal_azimuth(df_filtered)
|
||||
logging.info(f"Drone appears to be flying mainly on the courses {azi1:.2f} degrees and {azi2:.2f} degrees")
|
||||
|
||||
df_filtered["rolling_course_azimuth"] = (
|
||||
df_filtered["course_azimuth"].rolling(azimuth_window, center=True).apply(lambda x: circ_median(x), raw=True)
|
||||
)
|
||||
|
||||
df_filtered = df_filtered[
|
||||
(df_filtered["rolling_course_azimuth"] < azi1 + azimuth_filter)
|
||||
& (df_filtered["rolling_course_azimuth"] > azi1 - azimuth_filter)
|
||||
| (df_filtered["rolling_course_azimuth"] < azi2 + azimuth_filter)
|
||||
& (df_filtered["rolling_course_azimuth"] > azi2 - azimuth_filter)
|
||||
]
|
||||
|
||||
return df_filtered, df
|
||||
|
||||
|
||||
def mCount_max(data_dict: dict[int, float]) -> tuple[int, int]:
|
||||
"""
|
||||
Finds the start and end of the longest monotonic sequence in a dictionary of floats, typically used
|
||||
to identify a series of continuous altitude measurements. The first and last are retained.
|
||||
|
||||
Parameters:
|
||||
data_dict (Dict[int, float]): Dictionary with sequential numeric keys and numeric values representing measures
|
||||
such as altitude.
|
||||
|
||||
Returns:
|
||||
tuple: Start and end indices of the longest monotonic sequence in the dictionary.
|
||||
"""
|
||||
if len(data_dict) < 2:
|
||||
raise ValueError("Dictionary must contain at least two values")
|
||||
if list(data_dict.keys()) != list(range(1, len(data_dict) + 1)):
|
||||
raise ValueError("Keys must be sequential integers starting from 1")
|
||||
poscount = 0
|
||||
negcount = 0
|
||||
max_pos_count = 0
|
||||
max_pos_transect = 0
|
||||
max_neg_count = 0
|
||||
max_neg_transect = 0
|
||||
pos_start = 0
|
||||
neg_start = 0
|
||||
|
||||
for i in range(2, len(data_dict) + 1):
|
||||
if data_dict[i] >= data_dict[i - 1]:
|
||||
poscount += 1
|
||||
negcount = 0
|
||||
elif data_dict[i] < data_dict[i - 1]:
|
||||
negcount += 1
|
||||
poscount = 0
|
||||
|
||||
if max_pos_count < poscount:
|
||||
max_pos_count = poscount
|
||||
max_pos_transect = i
|
||||
pos_start = i - poscount
|
||||
elif max_neg_count < negcount:
|
||||
max_neg_count = negcount
|
||||
max_neg_transect = i
|
||||
neg_start = i - negcount
|
||||
|
||||
if max_pos_count > 0 or max_neg_count > 0:
|
||||
if max_pos_count >= max_neg_count:
|
||||
return pos_start, max_pos_transect
|
||||
else:
|
||||
return neg_start, max_neg_transect
|
||||
else:
|
||||
return 0, 0
|
||||
|
||||
|
||||
def largest_monotonic_transect_series(
|
||||
df: pd.DataFrame, transect_col: str = "transect_num", alt_col: str = "height_ato"
|
||||
) -> tuple[pd.DataFrame, int, int]:
|
||||
"""
|
||||
Filters the input dataframe to include only the largest continuous series of transects based on
|
||||
monotonic altitude changes.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe with transect and altitude information.
|
||||
transect_col (str): Column name for transect numbers. Default is "transect_num".
|
||||
alt_col (str): Column name for altitude data. Default is "height_ato".
|
||||
|
||||
Returns:
|
||||
tuple: The filtered dataframe, start transect, and end transect of the largest monotonic series.
|
||||
"""
|
||||
df = add_transect_azimuth_switches(df) # course switches
|
||||
alt_dict = dict(df.groupby(transect_col)[alt_col].mean())
|
||||
starttransect, endtransect = mCount_max(alt_dict) # type: ignore
|
||||
# filter to the biggest monotonic series of values
|
||||
df = df[(df[transect_col] >= starttransect) & (df[transect_col] <= endtransect)]
|
||||
logging.info(
|
||||
f"Parsed a flight of {len(np.unique(df[transect_col]))} transects from {alt_dict[starttransect]:.0f}m"
|
||||
f" to {alt_dict[endtransect]:.0f}m between {df['timestamp'].iloc[0]} and {df['timestamp'].iloc[-1]}",
|
||||
)
|
||||
return df, starttransect, endtransect
|
||||
|
||||
|
||||
def monotonic_transect_groups(
|
||||
df: pd.DataFrame, transect_col: str = "transect_num", alt_col: str = "height_ato"
|
||||
) -> tuple[pd.DataFrame, dict[int, str]]:
|
||||
"""
|
||||
Groups transects into a dict of monotonic transect sequences based on altitude, facilitating analysis of continuous
|
||||
flight patterns. Current behaviour is to reuse end transects of previous sequences as the start of the next.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe with transect and altitude information.
|
||||
transect_col (str): Column name for transect numbers. Default is "transect_num".
|
||||
alt_col (str): Column name for altitude data. Default is "height_ato".
|
||||
|
||||
Returns:
|
||||
tuple: The dataframe with a new 'group' column indicating the monotonic group ID, and a
|
||||
dictionary mapping transects to group IDs.
|
||||
"""
|
||||
|
||||
df = add_transect_azimuth_switches(df)
|
||||
alt_dict = dict(df.groupby(transect_col)[alt_col].mean())
|
||||
|
||||
group_dict = {}
|
||||
previous_altitude = None
|
||||
current_group = 1
|
||||
previous_trend = None
|
||||
first_transect_in_series = True
|
||||
|
||||
for transect, altitude in alt_dict.items():
|
||||
if previous_altitude is None:
|
||||
group_dict[transect] = f"Group_{current_group}"
|
||||
else:
|
||||
if altitude == previous_altitude:
|
||||
raise ValueError("Error: altitude is the same as the previous transect")
|
||||
elif altitude > previous_altitude:
|
||||
current_trend = "ascending"
|
||||
else:
|
||||
current_trend = "descending"
|
||||
|
||||
if current_trend != previous_trend and previous_trend is not None and not first_transect_in_series:
|
||||
current_group += 1 # Increment group counter
|
||||
first_transect_in_series = True
|
||||
if current_trend == previous_trend or previous_trend is None:
|
||||
# handles edge case where someone flies up, flies down one transect and then up again (yes really)
|
||||
first_transect_in_series = False
|
||||
group_dict[transect] = f"Group_{current_group}"
|
||||
previous_trend = current_trend
|
||||
previous_altitude = altitude
|
||||
df["group"] = df["transect_num"].map(group_dict)
|
||||
|
||||
return df, group_dict
|
||||
|
||||
|
||||
def remove_non_transects(
|
||||
df: pd.DataFrame,
|
||||
chain_length: int = 70,
|
||||
azimuth_tolerance: int = 10,
|
||||
elevation_tolerance: int = 40,
|
||||
smoothing_window: int = 5,
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""
|
||||
Filters the dataframe to remove segments not matching the criteria for being considered as transects,
|
||||
based on course azimuth and elevation and segment length.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe.
|
||||
chain_length (int): Minimum number of consecutive points to be considered a transect. Default is 70.
|
||||
azimuth_tolerance (int): Tolerance for deviation from modal course azimuths. Default is 10.
|
||||
elevation_tolerance (int): Tolerance for deviation from modal course elevation. Default is 40.
|
||||
smoothing_window (int): Window size for rolling median smoothing of courses. Default is 5. Rolling median
|
||||
avoids single point errors.
|
||||
|
||||
Returns:
|
||||
tuple: Dataframes of removed segments and retained segments that fit transect criteria.
|
||||
"""
|
||||
|
||||
def get_true_runs(mask):
|
||||
enumerated_mask = list(enumerate(mask)) # add index numbers to mask
|
||||
# group consecutive true/false values
|
||||
groups = groupby(enumerated_mask, key=lambda x: x[1])
|
||||
# retain groups of True values
|
||||
true_runs = [list(group) for key, group in groups if key]
|
||||
return true_runs
|
||||
|
||||
def split_runs_on_azimuth_inversion(df, runs, azimuth_inversion_threshold=120):
|
||||
split_runs = []
|
||||
for run in runs:
|
||||
last_azimuth = df.iloc[run[0][0]]["smoothed_course_azimuth"]
|
||||
current_run = [run[0]]
|
||||
for point in run[1:]:
|
||||
current_azimuth = df.iloc[point[0]]["smoothed_course_azimuth"]
|
||||
if min_angular_displacement(current_azimuth, last_azimuth) > azimuth_inversion_threshold:
|
||||
split_runs.append(current_run)
|
||||
current_run = [point]
|
||||
else:
|
||||
current_run.append(point)
|
||||
last_azimuth = current_azimuth
|
||||
split_runs.append(current_run)
|
||||
return split_runs
|
||||
|
||||
# Create new columns for filtering reasons, initialized with False
|
||||
df_removed = df.copy()
|
||||
df_removed["filtered_by_azimuth"] = False
|
||||
df_removed["filtered_by_elevation"] = False
|
||||
df_removed["filtered_by_chain"] = False
|
||||
|
||||
# Apply azimuth filter
|
||||
modal_course_azimuths = bimodal_azimuth(df, course_col="course_azimuth")
|
||||
modal_elevation_azimuths = bimodal_elevation(df, course_col="course_elevation")
|
||||
|
||||
# Apply rolling median to course azimuth and elevation and store them in new columns
|
||||
df_removed["smoothed_course_azimuth"] = (
|
||||
df_removed["course_azimuth"]
|
||||
.rolling(smoothing_window, center=True)
|
||||
.apply(lambda x: circmean(x, 360, 0), raw=True)
|
||||
.fillna(df_removed["course_azimuth"], inplace=False)
|
||||
)
|
||||
|
||||
df_removed["smoothed_course_elevation"] = (
|
||||
df_removed["course_elevation"]
|
||||
.rolling(smoothing_window, center=True)
|
||||
.apply(lambda x: circmean(x, 360, 0), raw=True)
|
||||
.fillna(df_removed["course_elevation"], inplace=False)
|
||||
)
|
||||
|
||||
azimuth_mask = df_removed["smoothed_course_azimuth"].apply(
|
||||
lambda x: any([min_angular_displacement(x, mode) <= azimuth_tolerance for mode in modal_course_azimuths])
|
||||
)
|
||||
|
||||
df_removed.loc[~azimuth_mask, "filtered_by_azimuth"] = True
|
||||
|
||||
# Apply elevation filter
|
||||
elevation_mask = df_removed["smoothed_course_elevation"].apply(
|
||||
lambda x: any([min_angular_displacement(x, mode) <= elevation_tolerance for mode in modal_elevation_azimuths])
|
||||
)
|
||||
df_removed.loc[~elevation_mask, "filtered_by_elevation"] = True
|
||||
|
||||
# Combined mask for azimuth and elevation
|
||||
mask = azimuth_mask & elevation_mask
|
||||
|
||||
true_runs = get_true_runs(mask)
|
||||
split_runs = split_runs_on_azimuth_inversion(df_removed, true_runs)
|
||||
chain_mask = [len(run) >= chain_length for run in split_runs]
|
||||
chain_filtered_runs = [run for run, filter_by_chain in zip(split_runs, chain_mask, strict=False) if filter_by_chain]
|
||||
chain_filtered_indices = [point[0] for run in chain_filtered_runs for point in run]
|
||||
df_removed.loc[df_removed.index.difference(chain_filtered_indices), "filtered_by_chain"] = True
|
||||
|
||||
filtered_segments = [df_removed.iloc[run[0][0] : run[-1][0] + 1] for run in chain_filtered_runs]
|
||||
|
||||
df_retained = pd.concat(filtered_segments)
|
||||
df_removed = df_removed[~df_removed.index.isin(df_retained.index)]
|
||||
|
||||
return df_removed, df_retained
|
||||
|
||||
|
||||
def flatten_linear_plane(
|
||||
df: pd.DataFrame, alt_col: str = "height_ato", distance_filter: float = 10000
|
||||
) -> tuple[pd.DataFrame, float]:
|
||||
"""
|
||||
Transforms a 3D dataset into a linear plane, focusing on the largest contiguous dataset aligned with
|
||||
the plane's primary orientation. The new x coordinate is the distance along the plane, y is the distance
|
||||
perpendicular to the plane (useful only for deviation), and z is the altitude.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): Input dataframe with "utm_easting", "utm_northing", and "height_ato" columns.
|
||||
alt_col (str): Column name for altitude data. Default is "height_ato".
|
||||
distance_filter (float): Threshold for filtering points based on their distance from the regression line.
|
||||
|
||||
Returns:
|
||||
tuple: Modified dataframe with new 'x', 'y', and 'z' columns representing transformed coordinates,
|
||||
and one of the plane's angles of rotation (from N, 0-360) in degrees.
|
||||
"""
|
||||
|
||||
def orthogonal_distance_regression(df: pd.DataFrame) -> tuple[pd.DataFrame, np.ndarray[Any, Any]]:
|
||||
"""
|
||||
Perform orthogonal distance regression on the given DataFrame.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): DataFrame containing "utm_easting" and "utm_northing" columns.
|
||||
|
||||
Returns:
|
||||
tuple[pd.DataFrame, np.ndarra[Any,Any]]]: Updated DataFrame with "distance_from_fit" column
|
||||
and the fitted parameters (slope, intercept).
|
||||
"""
|
||||
|
||||
def linear_reg_equation(B, x):
|
||||
return B[0] * x + B[1] # y = mx + c
|
||||
|
||||
required_columns = ["utm_easting", "utm_northing"]
|
||||
if not all(col in df.columns for col in required_columns):
|
||||
raise ValueError(f"DataFrame must contain columns: {required_columns}")
|
||||
model = odr.Model(linear_reg_equation)
|
||||
data = odr.Data(df["utm_easting"], df["utm_northing"])
|
||||
|
||||
INITIAL_BETA = [1, 0] # Initial guess of slope=1, intercept=0
|
||||
odr_instance = odr.ODR(data, model, beta0=INITIAL_BETA)
|
||||
fit = odr_instance.run()
|
||||
|
||||
if fit.stopreason[0] == "Iteration limit reached":
|
||||
raise RuntimeError("ODR fitting failed to converge")
|
||||
|
||||
slope, intercept = fit.beta
|
||||
df = df.assign(
|
||||
distance_from_fit=abs((slope * df["utm_easting"] - df["utm_northing"] + intercept) / np.sqrt(slope**2 + 1))
|
||||
)
|
||||
|
||||
return df, fit.beta
|
||||
|
||||
df, coefs2D = orthogonal_distance_regression(df)
|
||||
df = df.loc[df["distance_from_fit"] < distance_filter, :]
|
||||
df, coefs2D = orthogonal_distance_regression(df) # this is intentionally done twice
|
||||
df = df.loc[df["distance_from_fit"] < distance_filter, :]
|
||||
rotation = np.arctan(coefs2D[0])
|
||||
df.loc[:, "x"] = (df["utm_easting"] - df["utm_easting"].min()) * np.cos(-rotation) - (
|
||||
df["utm_northing"] - df["utm_northing"].min()
|
||||
) * np.sin(-rotation)
|
||||
df.loc[:, "y"] = (df["utm_easting"] - df["utm_easting"].min()) * np.sin(-rotation) + (
|
||||
df["utm_northing"] - df["utm_northing"].min()
|
||||
) * np.cos(-rotation)
|
||||
df.loc[:, "z"] = df[alt_col]
|
||||
|
||||
plane_angle = (np.pi / 2) - np.arctan(coefs2D[0])
|
||||
plane_angle = np.degrees(plane_angle)
|
||||
return df, plane_angle
|
||||
|
||||
|
||||
## Functions for circular/spiral flights ##
|
||||
|
||||
|
||||
def circle_deviation(df: pd.DataFrame, x_col: str, y_col: str) -> tuple[pd.DataFrame, float, float, float]:
|
||||
"""
|
||||
Calculates the deviation of points from a fitted circle and their azimuth angles.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe containing the data.
|
||||
x (str): Column name for the x-coordinate.
|
||||
y (str): Column name for the y-coordinate.
|
||||
|
||||
Returns:
|
||||
Tuple[pd.Dataframe, float, float, float]: A tuple containing the modified dataframe with azimuth angles
|
||||
and deviations,the radius of the fitted circle, and the coordinates of the circle's center.
|
||||
"""
|
||||
df = df.copy()
|
||||
required_columns = [x_col, y_col]
|
||||
if not all(col in df.columns for col in required_columns):
|
||||
raise ValueError(f"DataFrame must contain columns: {required_columns}")
|
||||
|
||||
x = df[x_col]
|
||||
y = df[y_col]
|
||||
|
||||
def midhalf(x): # select middle half of data to avoid edge effects
|
||||
return x.iloc[int(len(x) * 1 / 4) : int(len(x) * 3 / 4)]
|
||||
|
||||
x_filter, y_filter = midhalf(x), midhalf(y)
|
||||
|
||||
def func(params):
|
||||
xc, yc, r = params
|
||||
return np.sqrt((x - xc) ** 2 + (y - yc) ** 2) - r
|
||||
|
||||
x_m = np.mean(np.array(x_filter)) # initial guess for parameters
|
||||
y_m = np.mean(np.array(y_filter))
|
||||
r_m = np.mean(np.sqrt((np.array(x_filter) - x_m) ** 2 + (np.array(y_filter) - y_m) ** 2))
|
||||
|
||||
params0 = np.array([x_m, y_m, r_m])
|
||||
result = least_squares(func, params0)
|
||||
xc, yc, r = result.x
|
||||
|
||||
deviation = np.sqrt((x - xc) ** 2 + (y - yc) ** 2) - r
|
||||
|
||||
# output azimuth in radians with 0 at north and increasing clockwise thanks to modulos
|
||||
azimuth = np.degrees(np.arctan2(x - xc, y - yc) % (2 * np.pi))
|
||||
|
||||
df["circ_azimuth"] = azimuth
|
||||
df["circ_deviation"] = deviation
|
||||
|
||||
return df, r, xc, yc
|
||||
|
||||
|
||||
def recentre_azimuth(df: pd.DataFrame, r: float, x: str = "circ_azimuth", y: str = "ch4_normalised") -> pd.DataFrame:
|
||||
"""
|
||||
Recentres the azimuth angles based on the angle of maximum value and computes the distance along the
|
||||
circumference of the circle.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe.
|
||||
r (float): The radius of the circle.
|
||||
x (str): Column name for azimuth angles. Default is 'circ_azimuth'.
|
||||
y (str): Column name for the values used to find the maximum. Default is 'ch4_normalised'.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: The modified dataframe with centered azimuth angles and distances along the circumference.
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
def azimuth_of_max(df: pd.DataFrame, x: str = "circ_azimuth", y: str = "ch4_normalised") -> float:
|
||||
return df.loc[df[y].idxmax()][x] # type: ignore
|
||||
|
||||
centre_azimuth = azimuth_of_max(df, x, y)
|
||||
df["centred_azimuth"] = df[x] - centre_azimuth
|
||||
df.loc[df["centred_azimuth"] > 180, "centred_azimuth"] -= 360
|
||||
df.loc[df["centred_azimuth"] < -180, "centred_azimuth"] += 360
|
||||
df["circumference_distance"] = r * np.radians(df["centred_azimuth"])
|
||||
df["circumference_distance"] = df["circumference_distance"] - df["circumference_distance"].min()
|
||||
return df
|
||||
|
||||
|
||||
def drone_anemo_to_point_wind(
|
||||
df: pd.DataFrame, yaw_col: str, anemo_u_col: str, anemo_v_col: str, easting_col: str, northing_col: str
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Convert anemometer wind data from drone's coordinate system to Earth's coordinate system
|
||||
and calculate true wind speed and direction.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): Input DataFrame containing drone yaw, anemometer data, and drone speed.
|
||||
yaw_col (str): Column name for drone's yaw (in degrees, range [-180, 180]).
|
||||
anemo_u_col (str): Column name for anemometer U (wind speed in drone's X direction, from port to starboard).
|
||||
anemo_v_col (str): Column name for anemometer V (wind speed in drone's Y direction, from aft to nose).
|
||||
easting_col (str): Column name for drone's speed from west to east
|
||||
northing_col (str): Column name for drone's speed from south to north
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: DataFrame with calculated true wind speed ("windspeed") and true wind direction ("winddir").
|
||||
"""
|
||||
yaw_rad = np.deg2rad(df[yaw_col] % 360)
|
||||
rotated_U = df[anemo_u_col] * np.cos(yaw_rad) + df[anemo_v_col] * np.sin(yaw_rad)
|
||||
rotated_V = -df[anemo_u_col] * np.sin(yaw_rad) + df[anemo_v_col] * np.cos(yaw_rad)
|
||||
true_U = -rotated_U - df[easting_col]
|
||||
true_V = -rotated_V - df[northing_col]
|
||||
df["windspeed"] = np.sqrt(true_U**2 + true_V**2)
|
||||
df["winddir"] = np.degrees(np.arctan2(true_U, true_V)) % 360
|
||||
|
||||
return df
|
||||
317
src/gasflux/processing_pipelines.py
Normal file
317
src/gasflux/processing_pipelines.py
Normal file
@ -0,0 +1,317 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from scipy import stats
|
||||
|
||||
import pandas as pd
|
||||
import plotly.graph_objects as go
|
||||
import yaml
|
||||
|
||||
from src.gasflux import background,plotting,processing,reporting,interpolation,pre_processing,gas
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def read_csv(file_path: Path) -> pd.DataFrame:
|
||||
"""Read a CSV file and return a DataFrame."""
|
||||
try:
|
||||
return pd.read_csv(file_path)
|
||||
except FileNotFoundError:
|
||||
logger.exception(f"File not found: {file_path}")
|
||||
raise
|
||||
|
||||
|
||||
def load_config(config_path: Path) -> dict:
|
||||
"""Load a YAML config file and return a dictionary."""
|
||||
try:
|
||||
with open(config_path) as file:
|
||||
return yaml.safe_load(file)
|
||||
except FileNotFoundError:
|
||||
logger.exception(f"Config file not found: {config_path}")
|
||||
raise
|
||||
except yaml.YAMLError as e:
|
||||
logger.exception(f"Error parsing YAML config: {e}")
|
||||
raise
|
||||
|
||||
|
||||
class DataValidator: # TODO(me): decide whether to move this to preprocessing
|
||||
"""Validate the data before processing."""
|
||||
|
||||
def __init__(self, df: pd.DataFrame, config: dict) -> None:
|
||||
"""Initialise the validator with the DataFrame and config."""
|
||||
self.df = df
|
||||
self.required_cols = config["required_cols"].copy()
|
||||
self.required_cols.update(config["gases"])
|
||||
|
||||
def validate(self) -> None:
|
||||
"""Validate the data."""
|
||||
self._check_is_df()
|
||||
self._check_cols()
|
||||
self._check_dtypes()
|
||||
self._check_ranges()
|
||||
logger.info("Data validation passed")
|
||||
|
||||
def _check_is_df(self) -> None:
|
||||
"""Check that the input is a DataFrame."""
|
||||
if not isinstance(self.df, pd.DataFrame):
|
||||
logging.error("Input data is not a DataFrame.")
|
||||
raise ValueError("Input data is not a DataFrame.")
|
||||
|
||||
def _check_cols(self) -> None:
|
||||
"""Check that the required columns are present."""
|
||||
missing_cols = set(self.required_cols) - set(self.df.columns)
|
||||
if missing_cols:
|
||||
logging.error(f"Missing or mislabelled columns: {missing_cols}")
|
||||
raise ValueError(f"Missing or mislabelled columns: {missing_cols}")
|
||||
|
||||
def _check_dtypes(self) -> None:
|
||||
"""Check that the required columns are of the correct type and do not contain NaN values."""
|
||||
for col in self.required_cols:
|
||||
if col in self.df:
|
||||
if self.df[col].isna().any():
|
||||
logging.error(f"Column '{col}' contains NaN values.")
|
||||
raise ValueError(f"Column '{col}' contains NaN values.")
|
||||
if self.df[col].dtype != "float64":
|
||||
logging.error(f"Column '{col}' is not of type 'float64'.")
|
||||
raise ValueError(f"Column '{col}' is not of type 'float64'.")
|
||||
|
||||
def _check_ranges(self) -> None:
|
||||
"""Check that the required columns are within the specified ranges."""
|
||||
for col, (min_val, max_val) in self.required_cols.items():
|
||||
if col in self.df.columns and not self.df[col].between(min_val, max_val, inclusive="both").all():
|
||||
logging.error(f"Column '{col}' contains values out of range: {min_val} to {max_val}.")
|
||||
raise ValueError(f"Column '{col}' contains values out of range: {min_val} to {max_val}.")
|
||||
|
||||
|
||||
class BackgroundStrategy(ABC):
|
||||
def __init__(self, data_processor):
|
||||
self.data_processor = data_processor
|
||||
|
||||
@abstractmethod
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
|
||||
class AlgorithmicBaselineStrategy(BackgroundStrategy):
|
||||
def process(self):
|
||||
logger.info("Applying algorithmic background correction")
|
||||
for gas in self.data_processor.gases:
|
||||
(
|
||||
self.data_processor.df,
|
||||
self.data_processor.figs["background"][gas],
|
||||
self.data_processor.text[f"background_{gas}"],
|
||||
) = background.algorithmic_baseline(
|
||||
df=self.data_processor.df,
|
||||
gas=gas,
|
||||
algorithmic_baseline_settings=self.data_processor.config["algorithmic_baseline_settings"],
|
||||
)
|
||||
self.data_processor.df_std = self.data_processor.df.copy()
|
||||
|
||||
|
||||
class SensorStrategy(ABC):
|
||||
def __init__(self, data_processor):
|
||||
self.data_processor = data_processor
|
||||
|
||||
@abstractmethod
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
|
||||
class InSituSensorStrategy(SensorStrategy):
|
||||
def process(self):
|
||||
logger.info("Processing in-situ (point) data")
|
||||
for gas in self.data_processor.gases:
|
||||
self.data_processor.figs["scatter_3d"][gas] = plotting.scatter_3d(
|
||||
df=self.data_processor.df, color=gas, colorbar_title=f"{gas.upper()} flux (kg/m²/h)"
|
||||
)
|
||||
if SpatialProcessingStrategy == CurtainSpatialProcessingStrategy:
|
||||
self.data_processor.figs["windrose"] = plotting.windrose(self.data_processor.df, plot_transect=True)
|
||||
else:
|
||||
self.data_processor.figs["windrose"] = plotting.windrose(self.data_processor.df)
|
||||
self.data_processor.figs["wind_timeseries"] = plotting.time_series(
|
||||
self.data_processor.df, ys=["windspeed", "winddir"]
|
||||
)
|
||||
|
||||
|
||||
class SpatialProcessingStrategy(ABC):
|
||||
def __init__(self, data_processor):
|
||||
self.data_processor = data_processor
|
||||
|
||||
@abstractmethod
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
|
||||
class CurtainSpatialProcessingStrategy(SpatialProcessingStrategy):
|
||||
def process(self):
|
||||
logger.info("Applying curtain spatial processing")
|
||||
self.data_processor.dfs["original"] = self.data_processor.df.copy()
|
||||
self.data_processor.df, self.data_processor.start_transect, self.data_processor.end_transect = (
|
||||
processing.largest_monotonic_transect_series(self.data_processor.df)
|
||||
)
|
||||
self.data_processor.dfs["removed"] = self.data_processor.dfs["original"].loc[
|
||||
self.data_processor.dfs["original"].index.difference(self.data_processor.df.index)
|
||||
]
|
||||
self.data_processor.df, self.data_processor.plane_angle = processing.flatten_linear_plane(
|
||||
self.data_processor.df
|
||||
)
|
||||
self.data_processor.df = processing.wind_offset_correction(
|
||||
self.data_processor.df, self.data_processor.plane_angle
|
||||
)
|
||||
for gas_name in self.data_processor.gases:
|
||||
#计算通量
|
||||
self.data_processor.df = gas.gas_flux_column(self.data_processor.df, gas_name)
|
||||
self.data_processor.figs["scatter_3d"][gas_name].add_trace(
|
||||
go.Scatter3d(
|
||||
x=self.data_processor.dfs["removed"]["utm_easting"],
|
||||
y=self.data_processor.dfs["removed"]["utm_northing"],
|
||||
z=self.data_processor.dfs["removed"]["height_ato"],
|
||||
mode="markers",
|
||||
marker={"size": 2, "color": "black", "symbol": "circle", "opacity": 0.5},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class SpiralSpatialProcessingStrategy(SpatialProcessingStrategy):
|
||||
def process(self):
|
||||
logger.info("Applying spiral spatial processing")
|
||||
self.data_processor.dfs["original"] = self.data_processor.df.copy()
|
||||
# self.data_processor.df, self.data_processor.start_transect, self.data_processor.end_transect = (
|
||||
# gasflux.processing.largest_monotonic_transect_series(self.data_processor.df)
|
||||
|
||||
# no wind offset correction - assume wind is perpendicular to the spiral
|
||||
self.data_processor.dfs["removed"] = self.data_processor.dfs["original"].loc[
|
||||
self.data_processor.dfs["original"].index.difference(self.data_processor.df.index)
|
||||
]
|
||||
(
|
||||
self.data_processor.df,
|
||||
self.data_processor.circle_radius,
|
||||
self.data_processor.circle_center_x,
|
||||
self.data_processor.circle_center_y,
|
||||
) = processing.circle_deviation(self.data_processor.df, x_col="utm_easting", y_col="utm_northing")
|
||||
self.data_processor.df = processing.recentre_azimuth(
|
||||
self.data_processor.df, r=self.data_processor.circle_radius
|
||||
)
|
||||
self.data_processor.df["x"] = self.data_processor.df["circumference_distance"]
|
||||
for gas_name in self.data_processor.gases:
|
||||
self.data_processor.df = gas.gas_flux_column(self.data_processor.df, gas_name)
|
||||
self.data_processor.figs["scatter_3d"][gas_name].add_trace(
|
||||
go.Scatter3d(
|
||||
x=self.data_processor.dfs["removed"]["utm_easting"],
|
||||
y=self.data_processor.dfs["removed"]["utm_northing"],
|
||||
z=self.data_processor.dfs["removed"]["height_ato"],
|
||||
mode="markers",
|
||||
marker={"size": 2, "color": "black", "symbol": "circle", "opacity": 0.5},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class InterpolationStrategy(ABC):
|
||||
def __init__(self, data_processor):
|
||||
self.data_processor = data_processor
|
||||
|
||||
@abstractmethod
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
|
||||
class KrigingInterpolationStrategy(InterpolationStrategy):
|
||||
def process(self):
|
||||
logger.info("Applying kriging interpolation")
|
||||
for gas in self.data_processor.gases:
|
||||
(
|
||||
self.data_processor.output_vars["krig_parameters"][gas],
|
||||
self.data_processor.text[f"krig_output_{gas}"],
|
||||
self.data_processor.figs["contour"][gas],
|
||||
self.data_processor.figs["krig_grid"][gas],
|
||||
self.data_processor.figs["semivariogram"][gas],
|
||||
) = interpolation.ordinary_kriging(
|
||||
df=self.data_processor.df,
|
||||
x="x",
|
||||
y="height_ato",
|
||||
gas=gas,
|
||||
ordinary_kriging_settings=self.data_processor.config["ordinary_kriging_settings"],
|
||||
**self.data_processor.config["semivariogram_settings"],
|
||||
)
|
||||
logger.info(f"Kriged {gas}")
|
||||
|
||||
|
||||
class DataProcessor:
|
||||
def __init__(self, config: dict, df: pd.DataFrame):
|
||||
self.config: dict = config
|
||||
self.df: pd.DataFrame = df
|
||||
self.gases: list[str] = list(config["gases"].keys())
|
||||
self.processing_time = datetime.now()
|
||||
self.figs: dict = {
|
||||
"scatter_3d": {},
|
||||
"windrose": None,
|
||||
"wind_timeseries": None,
|
||||
"background": {},
|
||||
"contour": {},
|
||||
"krig_grid": {},
|
||||
"semivariogram": {},
|
||||
}
|
||||
self.text: dict = {}
|
||||
self.output_vars: dict = {"krig_parameters": {}, "std": {}}
|
||||
self.dfs: dict = {}
|
||||
self.reports: dict = {}
|
||||
|
||||
def strategy_selection(self):
|
||||
self.background_strategy: BackgroundStrategy
|
||||
if self.config["strategies"]["background"] == "algorithm":
|
||||
self.background_strategy = AlgorithmicBaselineStrategy(self)
|
||||
self.sensor_strategy: SensorStrategy
|
||||
if self.config["strategies"]["sensor"] == "insitu":
|
||||
self.sensor_strategy = InSituSensorStrategy(self)
|
||||
self.spatial_processing_strategy: SpatialProcessingStrategy
|
||||
if self.config["strategies"]["spatial"] == "curtain":
|
||||
self.spatial_processing_strategy = CurtainSpatialProcessingStrategy(self)
|
||||
if self.config["strategies"]["spatial"] == "spiral":
|
||||
self.spatial_processing_strategy = SpiralSpatialProcessingStrategy(self)
|
||||
self.interpolation_strategy: InterpolationStrategy
|
||||
if self.config["strategies"]["interpolation"] == "kriging":
|
||||
self.interpolation_strategy = KrigingInterpolationStrategy(self)
|
||||
|
||||
def process(self):
|
||||
self.df = pre_processing.add_utm(self.df)
|
||||
self.df = pre_processing.add_course(self.df)
|
||||
DataValidator(self.df, self.config).validate()
|
||||
self.background_strategy.process()
|
||||
self.sensor_strategy.process()
|
||||
self.spatial_processing_strategy.process()
|
||||
self.interpolation_strategy.process()
|
||||
|
||||
# Reporting
|
||||
for gas in self.gases:
|
||||
self.reports[gas] = reporting.mass_balance_report(
|
||||
krig_params=self.output_vars["krig_parameters"][gas],
|
||||
wind_fig=self.figs["wind_timeseries"],
|
||||
background_fig=self.figs["background"][gas],
|
||||
threed_fig=self.figs["scatter_3d"][gas],
|
||||
krig_fig=self.figs["contour"][gas],
|
||||
windrose_fig=self.figs["windrose"],
|
||||
)
|
||||
|
||||
# Collecting descriptive variables
|
||||
self.output_vars["std"]["windspeed"] = self.df["windspeed"].std()
|
||||
self.output_vars["std"]["windddir"] = stats.circstd(self.df["winddir"], high=360)
|
||||
for gas in self.gases:
|
||||
self.output_vars["std"][f"{gas}_background"] = self.df.loc[
|
||||
~self.df[f"{gas}_signal"], f"{gas}_normalised"
|
||||
].std()
|
||||
|
||||
|
||||
def process_main(data_file: Path, config_file: Path) -> None:
|
||||
"""Main function to run the pipeline."""
|
||||
config = load_config(config_file)
|
||||
name = data_file.stem
|
||||
df = read_csv(data_file)
|
||||
|
||||
processor = DataProcessor(config, df)
|
||||
processor.strategy_selection()
|
||||
processor.process()
|
||||
reporting.generate_reports(name, processor, config)
|
||||
logger.info("Processing complete")
|
||||
190
src/gasflux/qiya.py
Normal file
190
src/gasflux/qiya.py
Normal file
@ -0,0 +1,190 @@
|
||||
import requests
|
||||
import time
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
def get_pressure_at_location(lat, lon, altitude, date, time, max_retries=3, timeout=30):
|
||||
"""
|
||||
获取指定位置、时间、高度的气压
|
||||
|
||||
Args:
|
||||
lat: 纬度
|
||||
lon: 经度
|
||||
altitude: 海拔高度 (米)
|
||||
date: 日期 (格式: YYYY-MM-DD 或 YYYY/MM/DD)
|
||||
time: 时间 (格式: HH:MM 或 HH:MM:SS)
|
||||
max_retries: 最大重试次数
|
||||
timeout: 请求超时时间(秒)
|
||||
|
||||
Returns:
|
||||
float: 气压值 (hPa)，获取失败返回 None
|
||||
"""
|
||||
|
||||
# 标准化日期格式为 YYYY-MM-DD
|
||||
def normalize_date(d):
|
||||
"""将各种日期格式标准化为 YYYY-MM-DD"""
|
||||
if not d:
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
# 处理斜杠分隔符
|
||||
if "/" in d:
|
||||
d = d.replace("/", "-")
|
||||
|
||||
parts = d.split("-")
|
||||
if len(parts) == 3:
|
||||
year = parts[0]
|
||||
month = parts[1].zfill(2) # 确保月份是两位数
|
||||
day = parts[2].zfill(2) # 确保日期是两位数
|
||||
return f"{year}-{month}-{day}"
|
||||
else:
|
||||
# 如果格式不正确，返回今天的日期
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
date = normalize_date(date)
|
||||
|
||||
# 标准化时间格式为 HH:MM
|
||||
def normalize_time(t):
|
||||
"""将各种时间格式标准化为 HH:MM"""
|
||||
if not t or ":" not in t:
|
||||
return "12:00" # 默认中午12点
|
||||
|
||||
parts = t.split(":")
|
||||
if len(parts) >= 2:
|
||||
hour = parts[0].zfill(2) # 确保小时是两位数
|
||||
minute = parts[1].zfill(2) # 确保分钟是两位数
|
||||
return f"{hour}:{minute}"
|
||||
elif len(parts) == 1:
|
||||
hour = parts[0].zfill(2)
|
||||
return f"{hour}:00"
|
||||
else:
|
||||
return "12:00"
|
||||
|
||||
time = normalize_time(time)
|
||||
|
||||
url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
params = {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"start_date": date, # 格式: YYYY-MM-DD
|
||||
"end_date": date,
|
||||
"hourly": ["pressure_msl", "surface_pressure"],
|
||||
"timezone": "auto"
|
||||
}
|
||||
|
||||
# 创建带有重试机制的会话
|
||||
session = requests.Session()
|
||||
retry_strategy = Retry(
|
||||
total=max_retries,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
backoff_factor=1
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
try:
|
||||
print(f"正在获取位置 ({lat:.6f}, {lon:.6f}) 在 {date} {time} 的气压数据...")
|
||||
response = session.get(url, params=params, timeout=timeout)
|
||||
|
||||
# 检查响应状态
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
# 检查API错误
|
||||
if "error" in data:
|
||||
print(f"API错误: {data['error']}")
|
||||
return None
|
||||
|
||||
# 解析气压数据
|
||||
if data and "hourly" in data:
|
||||
times = data["hourly"]["time"]
|
||||
pressures = data["hourly"]["surface_pressure"] # 地表气压
|
||||
|
||||
if not times or not pressures:
|
||||
print("未找到气压数据")
|
||||
return None
|
||||
|
||||
# 根据时间找到对应气压
|
||||
target_time = f"{date}T{time}"
|
||||
if target_time in times:
|
||||
idx = times.index(target_time)
|
||||
pressure = pressures[idx]
|
||||
print(f"成功获取气压: {pressure} hPa")
|
||||
return pressure
|
||||
else:
|
||||
print(f"在数据中未找到时间: {target_time}")
|
||||
print(f"可用时间范围: {times[0]} 到 {times[-1]}")
|
||||
return None
|
||||
else:
|
||||
print("API响应中没有hourly数据")
|
||||
return None
|
||||
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
print(f"网络连接错误: {e}")
|
||||
print("请检查网络连接或稍后重试")
|
||||
return None
|
||||
except requests.exceptions.Timeout as e:
|
||||
print(f"请求超时: {e}")
|
||||
print(f"已重试 {max_retries} 次，请检查网络连接")
|
||||
return None
|
||||
except requests.exceptions.HTTPError as e:
|
||||
print(f"HTTP错误: {e}")
|
||||
return None
|
||||
except ValueError as e:
|
||||
print(f"数据解析错误: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"未知错误: {e}")
|
||||
return None
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def batch_get_pressure(data_list):
|
||||
"""
|
||||
批量获取多个位置的气压数据
|
||||
|
||||
Args:
|
||||
data_list: 包含 (lat, lon, altitude, date, time) 元组的列表
|
||||
|
||||
Returns:
|
||||
list: 气压值列表
|
||||
"""
|
||||
results = []
|
||||
for i, (lat, lon, alt, date, time) in enumerate(data_list):
|
||||
print(f"\n处理第 {i+1} 个位置...")
|
||||
pressure = get_pressure_at_location(lat, lon, alt, date, time)
|
||||
results.append(pressure)
|
||||
if pressure is not None:
|
||||
print(f"位置 {i+1}: {pressure} hPa")
|
||||
else:
|
||||
print(f"位置 {i+1}: 获取失败")
|
||||
|
||||
# 添加短暂延迟，避免请求过于频繁
|
||||
if i < len(data_list) - 1:
|
||||
time.sleep(0.5)
|
||||
|
||||
return results
|
||||
|
||||
# 使用示例
|
||||
if __name__ == "__main__":
|
||||
print("=== 气压数据获取工具 ===\n")
|
||||
|
||||
# 单个位置示例
|
||||
print("1. 单个位置查询:")
|
||||
pressure = get_pressure_at_location(
|
||||
lat=40.3491370, # 纽约纬度
|
||||
lon=115.7855289, # 纽约经度 (西经)
|
||||
altitude=435.789, # 海拔10米
|
||||
date="2016-02-12",
|
||||
time="08:00" # HH:MM格式
|
||||
)
|
||||
|
||||
if pressure is not None:
|
||||
print(f"纽约当前气压: {pressure} hPa")
|
||||
else:
|
||||
print("获取纽约气压数据失败")
|
||||
|
||||
130
src/gasflux/reporting.py
Normal file
130
src/gasflux/reporting.py
Normal file
@ -0,0 +1,130 @@
|
||||
"""This module provides functions for generating mass balance reports."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import plotly.graph_objects as go
|
||||
from jinja2 import Template
|
||||
from plotly.io import to_html
|
||||
from datetime import datetime
|
||||
|
||||
import yaml
|
||||
|
||||
import logging
|
||||
from . import plotting
|
||||
|
||||
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def mass_balance_report(
|
||||
krig_params: dict,
|
||||
wind_fig: go.Figure,
|
||||
background_fig: go.Figure,
|
||||
threed_fig: go.Figure,
|
||||
krig_fig: go.Figure,
|
||||
windrose_fig: go.Figure,
|
||||
) -> str:
|
||||
"""Generate a mass balance report."""
|
||||
template_path = Path(__file__).parent / "templates" / "mass_balance_template.html"
|
||||
|
||||
# Convert the figures to HTML
|
||||
plot_htmls = {}
|
||||
for name, fig in zip(
|
||||
["3D", "krig", "windrose", "wind", "background"],
|
||||
[threed_fig, krig_fig, windrose_fig, wind_fig, background_fig],
|
||||
strict=False,
|
||||
):
|
||||
if fig:
|
||||
plot_htmls[name] = to_html(fig, full_html=False)
|
||||
else:
|
||||
plot_htmls[name] = plotting.blank_figure()
|
||||
|
||||
summary_data = {
|
||||
"Estimated flux": f"{krig_params.get('volume', 0):.3f} kgh⁻¹",
|
||||
}
|
||||
|
||||
with Path.open(template_path) as f:
|
||||
template_content = f.read()
|
||||
|
||||
template = Template(template_content)
|
||||
return template.render(
|
||||
title="Mass Balance Report",
|
||||
summary_data=summary_data,
|
||||
threeD=plot_htmls["3D"],
|
||||
krig=plot_htmls["krig"],
|
||||
windrose=plot_htmls["windrose"],
|
||||
wind=plot_htmls["wind"],
|
||||
background=plot_htmls["background"],
|
||||
)
|
||||
|
||||
|
||||
def generate_reports(name: str, processor, config: dict):
|
||||
"""
|
||||
Generates reports, configuration files, and processed output variables for gasflux processing runs.
|
||||
|
||||
Parameters:
|
||||
name (str): The name identifier for the current processing run.
|
||||
processor (object): The processing object containing report data and output variables.
|
||||
config (dict): Configuration dictionary used for processing.
|
||||
"""
|
||||
output_dir = Path(config["output_dir"]).expanduser()
|
||||
processing_time = datetime.now()
|
||||
output_path = output_dir / name / processing_time.strftime("%Y-%m-%d_%H-%M-%S-%f_processing_run")
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save reports
|
||||
for gas, report in processor.reports.items():
|
||||
report_path = output_path / f"{name}_{gas}_report.html"
|
||||
with open(report_path, "w", encoding="utf-8") as file:
|
||||
file.write(report)
|
||||
|
||||
# Save config
|
||||
header = f"# Gasflux output config for file {name} from processing run at {processing_time}\n"
|
||||
config_path = output_path / f"{name}_config.yaml"
|
||||
with open(config_path, "w") as file:
|
||||
file.write(header)
|
||||
yaml.safe_dump(config, file)
|
||||
|
||||
# Save DataFrame to CSV
|
||||
if hasattr(processor, 'df') and processor.df is not None:
|
||||
csv_path = output_path / f"{name}_data.csv"
|
||||
processor.df.to_csv(csv_path, index=False)
|
||||
logger.info(f"DataFrame saved to {csv_path}")
|
||||
|
||||
# Save output variables
|
||||
output_vars = processor.output_vars
|
||||
# output_vars = delete_large_arrays(output_vars, threshold_size=50)
|
||||
header = (
|
||||
f"# Gasflux output variables for file {name} from processing run at {processing_time}\n"
|
||||
)
|
||||
filename = output_path / f"{name}_output_vars.json"
|
||||
with open(filename, "w") as file:
|
||||
file.write(header)
|
||||
json.dump(
|
||||
output_vars, file, default=lambda item: item.tolist() if isinstance(item, np.ndarray) else item, indent=4
|
||||
)
|
||||
logger.info(f"Processing run saved to {output_path}")
|
||||
|
||||
|
||||
def delete_large_arrays(output_vars: dict, threshold_size: int) -> dict:
|
||||
"""
|
||||
Iterate through the output_vars dictionary and replace large numpy arrays
|
||||
with their metadata (e.g., shape and data type).
|
||||
|
||||
Parameters:
|
||||
output_vars (dict): The dictionary containing output data including potential numpy arrays.
|
||||
threshold_size (int): The number of elements above which an array is considered large.
|
||||
"""
|
||||
del_keys = []
|
||||
for key, value in output_vars.items():
|
||||
if isinstance(value, dict):
|
||||
output_vars[key] = delete_large_arrays(value, threshold_size) # recursive
|
||||
elif isinstance(value, np.ndarray):
|
||||
if value.size > threshold_size:
|
||||
del_keys.append(key)
|
||||
for key in del_keys:
|
||||
del output_vars[key]
|
||||
return output_vars
|
||||
BIN
src/gasflux/resources/model.pkl
Normal file
BIN
src/gasflux/resources/model.pkl
Normal file
Binary file not shown.
142
src/gasflux/run_example.py
Normal file
142
src/gasflux/run_example.py
Normal file
@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GasFlux 完整处理示例脚本
|
||||
|
||||
这个脚本演示了完整的GasFlux处理流程：
|
||||
1. 使用data_processor.py处理原始Excel数据
|
||||
2. GasFlux数据验证和处理：
|
||||
- 背景校正
|
||||
- 空间处理
|
||||
- 克里金插值
|
||||
- 生成报告
|
||||
|
||||
运行方法：
|
||||
python run_example.py input.xlsx
|
||||
python run_example.py input.xlsx --output processed_data.csv
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
# 导入同一包内的模块
|
||||
# 优先尝试相对导入（当作为包的一部分运行时）
|
||||
# 如果失败，则尝试绝对导入（当直接运行脚本时）
|
||||
try:
|
||||
# 尝试相对导入
|
||||
from .processing_pipelines import process_main
|
||||
from .data_processor import process_file
|
||||
print("✅ 成功导入GasFlux模块（相对导入）")
|
||||
except ImportError:
|
||||
try:
|
||||
# 相对导入失败，尝试绝对导入
|
||||
from src.gasflux.processing_pipelines import process_main
|
||||
from src.gasflux.data_processor import process_file
|
||||
print("✅ 成功导入GasFlux模块（绝对导入）")
|
||||
except ImportError as e:
|
||||
print(f"❌ 导入GasFlux模块失败: {e}")
|
||||
print("请确保GasFlux包结构完整，或从项目根目录运行")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数：运行完整的数据处理和GasFlux分析流程"""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="GasFlux完整处理流程：Excel数据预处理 + 通量分析",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
使用示例:
|
||||
python run_example.py data.xlsx # 基本使用
|
||||
python run_example.py data.xlsx --output result.csv # 指定输出文件名
|
||||
python run_example.py data.xlsx --no-gasflux # 仅预处理，不进行通量分析
|
||||
|
||||
处理步骤:
|
||||
1. 使用data_processor.py处理Excel文件
|
||||
2. 对处理后的数据进行GasFlux通量分析
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('input_file', help='输入的Excel文件路径 (.xlsx 或 .xls)')
|
||||
parser.add_argument('-o', '--output', help='预处理后的CSV输出文件名（可选）')
|
||||
parser.add_argument('--no-gasflux', action='store_true',
|
||||
help='仅执行数据预处理，跳过GasFlux通量分析')
|
||||
parser.add_argument('-c', '--config', help='GasFlux配置文件路径（可选，默认使用内置配置）')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=== GasFlux 完整处理流程 ===\n")
|
||||
|
||||
# 检查输入文件
|
||||
input_path = Path(args.input_file)
|
||||
if not input_path.exists():
|
||||
print(f"❌ 错误：输入文件不存在: {input_path}")
|
||||
return
|
||||
|
||||
if input_path.suffix.lower() not in ['.xlsx', '.xls']:
|
||||
print(f"❌ 错误：输入文件必须是Excel格式: {input_path}")
|
||||
return
|
||||
|
||||
# 确定输出文件名
|
||||
if args.output:
|
||||
processed_csv = Path(args.output)
|
||||
else:
|
||||
processed_csv = input_path.with_suffix('.processed.csv')
|
||||
|
||||
# 确定配置文件
|
||||
if args.config:
|
||||
config_file = Path(args.config)
|
||||
else:
|
||||
config_file = Path(__file__).parent / "data" / "gasflux_config.yaml"
|
||||
|
||||
print(f"输入Excel文件: {input_path}")
|
||||
print(f"预处理输出CSV: {processed_csv}")
|
||||
if not args.no_gasflux:
|
||||
print(f"GasFlux配置文件: {config_file}")
|
||||
print()
|
||||
|
||||
try:
|
||||
# 第一步：数据预处理
|
||||
print("🔄 第一步：数据预处理...")
|
||||
processed_df = process_file(str(input_path), str(processed_csv))
|
||||
print(f"✅ 数据预处理完成，输出文件: {processed_csv}")
|
||||
print()
|
||||
|
||||
# 如果不需要进行GasFlux分析，直接返回
|
||||
if args.no_gasflux:
|
||||
print("⏭️ 跳过GasFlux通量分析")
|
||||
return
|
||||
|
||||
# 第二步：GasFlux通量分析
|
||||
print("🔄 第二步：GasFlux通量分析...")
|
||||
|
||||
# 检查配置文件
|
||||
if not config_file.exists():
|
||||
print(f"❌ 错误：GasFlux配置文件不存在: {config_file}")
|
||||
return
|
||||
|
||||
print(f"使用数据文件: {processed_csv}")
|
||||
print(f"使用配置文件: {config_file}")
|
||||
print()
|
||||
|
||||
# 运行GasFlux处理流程
|
||||
process_main(processed_csv, config_file)
|
||||
print("\n✅ GasFlux通量分析完成！")
|
||||
|
||||
# 显示输出信息
|
||||
output_dir = Path("../../examples/basic_usage/5m") / processed_csv.stem / "processing_run"
|
||||
if output_dir.exists():
|
||||
print(f"\n📁 GasFlux输出目录: {output_dir}")
|
||||
print("📊 生成的文件:")
|
||||
for file in output_dir.rglob("*"):
|
||||
if file.is_file():
|
||||
print(f" - {file.name}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 处理失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
67
src/gasflux/templates/mass_balance_template.html
Normal file
67
src/gasflux/templates/mass_balance_template.html
Normal file
@ -0,0 +1,67 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
.grid-container {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, 1fr);
|
||||
grid-gap: 10px;
|
||||
height: 800px;
|
||||
}
|
||||
|
||||
.grid-item.double {
|
||||
grid-column: span 2;
|
||||
}
|
||||
|
||||
.grid-item {
|
||||
width: 100%;
|
||||
min-width: 300px;
|
||||
}
|
||||
|
||||
.grid-item .plot-container {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>{{title}}</h1>
|
||||
<table>
|
||||
<tbody>
|
||||
{% for key, value in summary_data.items() %}
|
||||
<tr>
|
||||
<td>{{key}}</td>
|
||||
<td>{{value}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="grid-container">
|
||||
<div class="grid-item">
|
||||
<div class="plot-container">
|
||||
{{ threeD|safe }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="grid-item double">
|
||||
<div class="plot-container">
|
||||
{{ krig|safe }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="grid-item">
|
||||
<div class="plot-container">
|
||||
{{ windrose|safe }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="grid-item double">
|
||||
<div class="plot-container">
|
||||
{{ wind|safe }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="grid-item double">
|
||||
<div class="plot-container">
|
||||
{{ background|safe }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
1001
src/gasflux/testdata/exampledata.csv
vendored
Normal file
1001
src/gasflux/testdata/exampledata.csv
vendored
Normal file
File diff suppressed because it is too large
Load Diff
95
src/gasflux/testdata/testconfig.yaml
vendored
Normal file
95
src/gasflux/testdata/testconfig.yaml
vendored
Normal file
@ -0,0 +1,95 @@
|
||||
# testconfig.yaml
|
||||
|
||||
horizontal_pixels: 500 # Width of the concentration map in pixels
|
||||
vertical_pixels: 100 # Height of the concentration map in pixels
|
||||
num_plumes: 10 # Number of Gaussian plumes
|
||||
groupiness: 0.5 # Groupiness of the plumes (0.0 to 1.0)
|
||||
spread: 0.1 # Spread of the plumes (0.0 to 1.0)
|
||||
wind_reference_height: 10 # Reference height for wind speed calculation (m)
|
||||
windspeed_avg: 5 # Average wind speed at 10m height (m/s)
|
||||
windspeed_rel_std: 0.2 # Relative standard deviation of wind speed (0.0 to inf), recommend 0.2-0.4
|
||||
surface_roughness: 0.1 # Surface roughness length (m)
|
||||
seed: 42 # Random seed for reproducibility
|
||||
simplex_octaves: 4 # Number of octaves for simplex noise (1 to inf, def 1)
|
||||
simplex_persistence: 0.7 # Persistence of simplex noise (0.0 to 1.0, def 0.5) - specifies the amplitude of each octave relative to the one below it
|
||||
simplex_lacunarity: 2.0 # Lacunarity of simplex noise (1.0 to inf, def 2.0) - specifies the frequency of each octave relative to the one below it
|
||||
winddir_avg: 0.0 # Average wind direction in degrees rel to plane (0 is CW)
|
||||
winddir_std: 10 # Standard deviation of wind direction in degrees
|
||||
timestamp: "2022-09-26 02:03:00"
|
||||
flight_time_seconds: 1000
|
||||
sample_frequency: 1
|
||||
start_coords:
|
||||
- 54.87667
|
||||
- 15.41
|
||||
transect_azimuth: 260 # the wind will start off 90 degrees CW to this azimuth, and is modified relative to that by the winddir_avg. 260 is a good value to test N problems
|
||||
sampling_altitude_ato_range:
|
||||
- -10 # negative values should be fine
|
||||
- 100
|
||||
sampling_horizontal_range:
|
||||
- 50
|
||||
- 950
|
||||
scene_altitude_range:
|
||||
- -20
|
||||
- 120
|
||||
scene_horizontal_range:
|
||||
- 0
|
||||
- 1000
|
||||
number_of_transects: 10
|
||||
gases:
|
||||
ch4:
|
||||
- 1.95
|
||||
- 10.0
|
||||
co2:
|
||||
- 380.0
|
||||
- 500.0
|
||||
c2h6:
|
||||
- 0.0
|
||||
- 1.0
|
||||
temperature: 10.0
|
||||
pressure: 1000.0
|
||||
|
||||
output_dir: ~/gasflux_reports
|
||||
|
||||
algorithmic_baseline_settings:
|
||||
algorithm: fastchrom
|
||||
|
||||
semivariogram_settings:
|
||||
model: spherical
|
||||
estimator: cressie
|
||||
n_lags: 20
|
||||
bin_func: even
|
||||
fit_method: lm
|
||||
maxlag: 100
|
||||
#fit_sigma: linear
|
||||
tolerance: 10
|
||||
azimuth: 0
|
||||
bandwidth: 20
|
||||
|
||||
ordinary_kriging_settings:
|
||||
min_points: 3
|
||||
max_points: 100
|
||||
grid_resolution: 500
|
||||
min_nodes: 10
|
||||
cut_ground: False
|
||||
y_min: ~
|
||||
|
||||
required_cols:
|
||||
latitude: [-90, 90]
|
||||
longitude: [-180, 180]
|
||||
height_ato: [-100, 500]
|
||||
windspeed: [0, 30]
|
||||
winddir: [0, 360]
|
||||
temperature: [-50, 60]
|
||||
pressure: [900, 1100]
|
||||
|
||||
filters:
|
||||
course_filter:
|
||||
azimuth_filter: 10
|
||||
azimuth_window: 5
|
||||
elevation_filter: 5
|
||||
|
||||
strategies:
|
||||
background: "algorithm"
|
||||
sensor: "insitu"
|
||||
spatial: "curtain"
|
||||
interpolation: "kriging"
|
||||
1001
src/gasflux/testdata/testdata.csv
vendored
Normal file
1001
src/gasflux/testdata/testdata.csv
vendored
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user