常规更新
This commit is contained in:
@ -12,122 +12,122 @@ from flask_cors import CORS
|
||||
from werkzeug.utils import secure_filename
|
||||
import yaml
|
||||
|
||||
# Shared utilities imported from shared.py
|
||||
# 从 shared.py 导入共享工具
|
||||
try:
|
||||
# Try relative import (when run as part of package)
|
||||
# 尝试相对导入（作为包的一部分运行时）
|
||||
from .shared import TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED, update_task_status as shared_update_task_status
|
||||
except ImportError:
|
||||
# Fallback to absolute import (when run directly)
|
||||
# 回退到绝对导入（直接运行时）
|
||||
from shared import TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED, update_task_status as shared_update_task_status
|
||||
|
||||
# Load configuration from INI file
|
||||
# 从 INI 文件加载配置
|
||||
try:
|
||||
from .config_reader import config_reader
|
||||
except ImportError:
|
||||
from config_reader import config_reader
|
||||
|
||||
# Blueprints will be imported after app initialization to avoid circular imports
|
||||
# 蓝图将在应用初始化后导入，以避免循环导入
|
||||
|
||||
# Environment-based configuration management
|
||||
# 基于环境的配置管理
|
||||
class Config:
|
||||
"""Configuration management using INI file with environment variable fallbacks."""
|
||||
"""使用 INI 文件并支持环境变量回退的配置管理"""
|
||||
|
||||
# Server configuration from config_reader
|
||||
# 从 config_reader 获取服务器配置
|
||||
HOST = config_reader.host
|
||||
PORT = config_reader.port
|
||||
DEBUG = config_reader.debug
|
||||
BASE_URL = config_reader.base_url
|
||||
|
||||
# Directory configuration
|
||||
BASE_DIR = None # Will be set dynamically
|
||||
# 目录配置
|
||||
BASE_DIR = None # 将动态设置
|
||||
UPLOAD_FOLDER_NAME = str(config_reader.uploads_path)
|
||||
OUTPUT_FOLDER_NAME = str(config_reader.outputs_path)
|
||||
|
||||
# File size limits (in bytes)
|
||||
# 文件大小限制（字节）
|
||||
MAX_CONTENT_LENGTH = config_reader.max_content_length
|
||||
|
||||
# Logging configuration
|
||||
# 日志配置
|
||||
LOG_LEVEL = config_reader.log_level.upper()
|
||||
LOG_FILE = config_reader.log_file
|
||||
|
||||
# CORS configuration (keeping environment fallback for now)
|
||||
# CORS 配置（暂时保留环境回退）
|
||||
CORS_ORIGINS = os.getenv('GASFLUX_CORS_ORIGINS', '*').split(',')
|
||||
|
||||
# Task management
|
||||
# 任务管理
|
||||
TASK_CLEANUP_INTERVAL = config_reader.task_cleanup_interval
|
||||
MAX_TASK_AGE = config_reader.max_task_age
|
||||
SUCCESSFUL_TASK_CLEANUP_AGE = config_reader.successful_task_cleanup_age
|
||||
FAILED_TASK_CLEANUP_AGE = config_reader.failed_task_cleanup_age
|
||||
|
||||
# Performance tuning
|
||||
# 性能调优
|
||||
THREADS = config_reader.threads
|
||||
CONNECTION_LIMIT = config_reader.connection_limit
|
||||
CHANNEL_TIMEOUT = config_reader.channel_timeout
|
||||
|
||||
# Database configuration
|
||||
# 数据库配置
|
||||
DB_PATH = config_reader.db_path if config_reader.db_path else None
|
||||
|
||||
# Persistence backend
|
||||
# 持久化后端
|
||||
TASK_PERSIST_BACKEND = config_reader.persist_backend
|
||||
|
||||
# Janitor configuration
|
||||
# 清理工具配置
|
||||
JANITOR_DRY_RUN = config_reader.janitor_dry_run
|
||||
|
||||
# Admin bootstrap key
|
||||
# 管理员引导密钥
|
||||
ADMIN_BOOTSTRAP_KEY = config_reader.admin_bootstrap_key
|
||||
|
||||
@classmethod
|
||||
def init_base_dir(cls):
|
||||
"""Initialize base directory based on environment."""
|
||||
"""根据环境初始化基础目录"""
|
||||
try:
|
||||
if getattr(sys, 'frozen', False):
|
||||
# Running in PyInstaller bundle
|
||||
# 在 PyInstaller 打包环境中运行
|
||||
cls.BASE_DIR = Path(sys.executable).parent
|
||||
else:
|
||||
# Running in normal Python environment
|
||||
# 在正常 Python 环境中运行
|
||||
cls.BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
except:
|
||||
# Fallback to current working directory
|
||||
# 回退到当前工作目录
|
||||
cls.BASE_DIR = Path.cwd()
|
||||
|
||||
# Initialize directories based on config
|
||||
# 根据配置初始化目录
|
||||
cls.init_directories()
|
||||
|
||||
@classmethod
|
||||
def init_directories(cls, output_dir=None):
|
||||
"""Initialize upload and output directories from configuration."""
|
||||
# Use paths from config_reader
|
||||
"""从配置初始化上传和输出目录"""
|
||||
# 使用 config_reader 中的路径
|
||||
uploads_path = config_reader.uploads_path
|
||||
outputs_path = config_reader.outputs_path
|
||||
|
||||
# Resolve relative paths to absolute if needed
|
||||
# 如需要，将相对路径解析为绝对路径
|
||||
if not uploads_path.is_absolute():
|
||||
uploads_path = cls.BASE_DIR / uploads_path
|
||||
if not outputs_path.is_absolute():
|
||||
outputs_path = cls.BASE_DIR / outputs_path
|
||||
|
||||
# Set the resolved paths
|
||||
# 设置已解析的路径
|
||||
cls.UPLOAD_FOLDER = uploads_path
|
||||
cls.OUTPUT_FOLDER = outputs_path
|
||||
|
||||
# Create directories
|
||||
# 创建目录
|
||||
cls.UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
cls.OUTPUT_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Directories initialized - Upload: {cls.UPLOAD_FOLDER}, Output: {cls.OUTPUT_FOLDER}")
|
||||
|
||||
# For backward compatibility, also set the old-style paths
|
||||
# 为向后兼容，同时设置旧式路径
|
||||
if output_dir:
|
||||
logger.warning("output_dir parameter is deprecated, use gasflux.ini [paths] section instead")
|
||||
|
||||
@classmethod
|
||||
def update_directories_from_config(cls, config_path=None):
|
||||
"""Update directories based on config file. (DEPRECATED: Use gasflux.ini instead)"""
|
||||
"""根据配置文件更新目录（已弃用：请改用 gasflux.ini）"""
|
||||
logger.warning("update_directories_from_config is deprecated. Output directories are now configured via gasflux.ini [paths] section.")
|
||||
# No longer reads output_dir from YAML config - directories are set from INI config in init_directories()
|
||||
# 不再从 YAML 配置读取 output_dir - 目录在 init_directories() 中从 INI 配置设置
|
||||
|
||||
@classmethod
|
||||
def get_log_level(cls):
|
||||
"""Get logging level from string."""
|
||||
"""从字符串获取日志级别"""
|
||||
levels = {
|
||||
'DEBUG': logging.DEBUG,
|
||||
'INFO': logging.INFO,
|
||||
@ -139,7 +139,7 @@ class Config:
|
||||
|
||||
@classmethod
|
||||
def to_dict(cls):
|
||||
"""Return configuration as dictionary for debugging."""
|
||||
"""以字典形式返回配置，用于调试"""
|
||||
return {
|
||||
'host': cls.HOST,
|
||||
'port': cls.PORT,
|
||||
@ -160,12 +160,12 @@ class Config:
|
||||
'channel_timeout': cls.CHANNEL_TIMEOUT
|
||||
}
|
||||
|
||||
# Initialize logging with environment-based configuration
|
||||
# 使用基于环境的配置初始化日志
|
||||
logging.basicConfig(
|
||||
level=Config.get_log_level(),
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(), # Console output
|
||||
logging.StreamHandler(), # 控制台输出
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger("gasflux_api")
|
||||
@ -173,7 +173,7 @@ logger.info("Basic logging initialized")
|
||||
|
||||
|
||||
def log_performance(func):
|
||||
"""Decorator to log function performance."""
|
||||
"""用于记录函数性能的装饰器"""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
start_time = time.time()
|
||||
@ -190,24 +190,24 @@ def log_performance(func):
|
||||
raise
|
||||
return wrapper
|
||||
|
||||
# Task status management
|
||||
# Task status constants and storage moved to shared.py
|
||||
# 任务状态管理
|
||||
# 任务状态常量和存储已移至 shared.py
|
||||
|
||||
def update_task_status(task_id, status, message=None, results=None, error=None, output_dir=None):
|
||||
"""Update task status using the shared implementation (writes to SQLite)."""
|
||||
"""使用共享实现更新任务状态（写入 SQLite）"""
|
||||
return shared_update_task_status(task_id, status, message=message, results=results, error=error, output_dir=output_dir)
|
||||
|
||||
|
||||
# Statistics and Monitoring
|
||||
# 统计和监控
|
||||
class APIStatsCollector:
|
||||
"""Collect and manage API statistics."""
|
||||
"""收集和管理 API 统计信息"""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = time.time()
|
||||
self.reset_stats()
|
||||
|
||||
def reset_stats(self):
|
||||
"""Reset all statistics."""
|
||||
"""重置所有统计信息"""
|
||||
self.stats = {
|
||||
'requests': {
|
||||
'total': 0,
|
||||
@ -238,42 +238,42 @@ class APIStatsCollector:
|
||||
}
|
||||
|
||||
def record_request(self, method, endpoint, status_code, response_time):
|
||||
"""Record an API request."""
|
||||
"""记录 API 请求"""
|
||||
self.stats['requests']['total'] += 1
|
||||
|
||||
# Method stats
|
||||
# 方法统计
|
||||
if method not in self.stats['requests']['by_method']:
|
||||
self.stats['requests']['by_method'][method] = 0
|
||||
self.stats['requests']['by_method'][method] += 1
|
||||
|
||||
# Endpoint stats
|
||||
# 端点统计
|
||||
if endpoint not in self.stats['requests']['by_endpoint']:
|
||||
self.stats['requests']['by_endpoint'][endpoint] = 0
|
||||
self.stats['requests']['by_endpoint'][endpoint] += 1
|
||||
|
||||
# Status stats
|
||||
status_category = str(status_code // 100 * 100) # 200, 400, 500, etc.
|
||||
# 状态统计
|
||||
status_category = str(status_code // 100 * 100) # 200, 400, 500, 等
|
||||
if status_category not in self.stats['requests']['by_status']:
|
||||
self.stats['requests']['by_status'][status_category] = 0
|
||||
self.stats['requests']['by_status'][status_category] += 1
|
||||
|
||||
# Response time stats
|
||||
# 响应时间统计
|
||||
self.stats['requests']['response_times'].append(response_time)
|
||||
|
||||
# Keep only last 1000 response times for memory efficiency
|
||||
# 仅保留最近 1000 个响应时间以节省内存
|
||||
if len(self.stats['requests']['response_times']) > 1000:
|
||||
self.stats['requests']['response_times'] = self.stats['requests']['response_times'][-1000:]
|
||||
|
||||
# Error tracking
|
||||
# 错误跟踪
|
||||
if status_code >= 400:
|
||||
self.stats['requests']['errors'] += 1
|
||||
|
||||
# Update performance stats
|
||||
# 更新性能统计
|
||||
self._update_performance_stats()
|
||||
|
||||
def record_task_status_change(self, old_status, new_status):
|
||||
"""Record task status changes."""
|
||||
if old_status == "unknown": # New task
|
||||
"""记录任务状态变化"""
|
||||
if old_status == "unknown": # 新任务
|
||||
self.stats['tasks']['total_created'] += 1
|
||||
|
||||
if new_status == TASK_STATUS_COMPLETED:
|
||||
@ -281,7 +281,7 @@ class APIStatsCollector:
|
||||
elif new_status == TASK_STATUS_FAILED:
|
||||
self.stats['tasks']['total_failed'] += 1
|
||||
|
||||
# Update status counts
|
||||
# 更新状态计数
|
||||
for status in [old_status, new_status]:
|
||||
if status in self.stats['tasks']['by_status']:
|
||||
if status == old_status and old_status != "unknown":
|
||||
@ -290,15 +290,15 @@ class APIStatsCollector:
|
||||
self.stats['tasks']['by_status'][new_status] += 1
|
||||
|
||||
def record_task_completion_time(self, completion_time):
|
||||
"""Record task completion time."""
|
||||
"""记录任务完成时间"""
|
||||
self.stats['tasks']['processing_times'].append(completion_time)
|
||||
|
||||
# Keep only last 100 processing times
|
||||
# 仅保留最近 100 个处理时间
|
||||
if len(self.stats['tasks']['processing_times']) > 100:
|
||||
self.stats['tasks']['processing_times'] = self.stats['tasks']['processing_times'][-100:]
|
||||
|
||||
def _update_performance_stats(self):
|
||||
"""Update performance statistics."""
|
||||
"""更新性能统计信息"""
|
||||
response_times = self.stats['requests']['response_times']
|
||||
if response_times:
|
||||
self.stats['performance']['avg_response_time'] = sum(response_times) / len(response_times)
|
||||
@ -308,15 +308,15 @@ class APIStatsCollector:
|
||||
self.stats['performance']['uptime_seconds'] = time.time() - self.start_time
|
||||
|
||||
def get_summary(self):
|
||||
"""Get a summary of current statistics."""
|
||||
"""获取当前统计摘要"""
|
||||
current_time = time.time()
|
||||
uptime = current_time - self.start_time
|
||||
|
||||
# Calculate rates
|
||||
# 计算速率
|
||||
requests_per_second = self.stats['requests']['total'] / max(uptime, 1)
|
||||
error_rate = (self.stats['requests']['errors'] / max(self.stats['requests']['total'], 1)) * 100
|
||||
|
||||
# Task completion rate
|
||||
# 任务完成率
|
||||
total_tasks_processed = self.stats['tasks']['total_completed'] + self.stats['tasks']['total_failed']
|
||||
task_success_rate = (self.stats['tasks']['total_completed'] / max(total_tasks_processed, 1)) * 100
|
||||
|
||||
@ -351,7 +351,7 @@ class APIStatsCollector:
|
||||
}
|
||||
|
||||
def _format_uptime(self, seconds):
|
||||
"""Format uptime in human readable format."""
|
||||
"""以人类可读格式格式化运行时间"""
|
||||
days, remainder = divmod(int(seconds), 86400)
|
||||
hours, remainder = divmod(remainder, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
@ -368,15 +368,15 @@ class APIStatsCollector:
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
# Global statistics collector
|
||||
# 全局统计收集器
|
||||
stats_collector = APIStatsCollector()
|
||||
|
||||
# get_task_status moved to shared.py
|
||||
# get_task_status 已移至 shared.py
|
||||
|
||||
# cleanup_old_tasks moved to shared.py
|
||||
# cleanup_old_tasks 已移至 shared.py
|
||||
|
||||
def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
"""Background task to process data asynchronously."""
|
||||
"""异步处理数据的后台任务"""
|
||||
# 确保后台线程里有 Flask 应用上下文
|
||||
with app.app_context():
|
||||
logger.info(f"Job {task_id}: Background processing started for task {task_id}")
|
||||
@ -385,7 +385,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
try:
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "开始处理数据...")
|
||||
|
||||
# 1. Load and override config FIRST
|
||||
# 1. 首先加载并覆盖配置
|
||||
logger.info(f"Job {task_id}: Loading configuration from {config_path}")
|
||||
config_start = time.time()
|
||||
|
||||
@ -397,34 +397,34 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
logger.error(f"Job {task_id}: Failed to load config from {config_path}: {str(e)}")
|
||||
raise
|
||||
|
||||
# Update directories based on config output_dir
|
||||
# 根据配置 output_dir 更新目录
|
||||
Config.update_directories_from_config(config_path)
|
||||
|
||||
# Sync app.config with updated directories
|
||||
# 将更新后的目录同步到 app.config
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
|
||||
# Task status persistence now uses SQLite only
|
||||
# JSON persistence has been disabled
|
||||
# 任务状态持久化现在仅使用 SQLite
|
||||
# JSON 持久化已禁用
|
||||
# from .shared import set_task_status_file_path, load_task_status_from_file
|
||||
# set_task_status_file_path(Config.OUTPUT_FOLDER / "task_status.json")
|
||||
# load_task_status_from_file()
|
||||
|
||||
# Update job directories to be under the correct config-based paths
|
||||
# 更新任务目录到正确的基于配置的路径下
|
||||
from pathlib import Path
|
||||
job_upload_dir = Path(Config.UPLOAD_FOLDER) / task_id
|
||||
job_output_dir = Path(Config.OUTPUT_FOLDER) / task_id
|
||||
job_upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
job_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Trigger an update to save output_dir to database
|
||||
# 触发更新将 output_dir 保存到数据库
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "目录已就绪", output_dir=str(job_output_dir))
|
||||
|
||||
# Move uploaded files to the correct config-based directories
|
||||
# 将上传的文件移动到正确的基于配置的目录
|
||||
try:
|
||||
import shutil
|
||||
|
||||
# Move data file to correct uploads directory
|
||||
# 移动数据文件到正确的上传目录
|
||||
if data_path.parent != job_upload_dir:
|
||||
new_data_path = job_upload_dir / data_path.name
|
||||
if data_path != new_data_path:
|
||||
@ -432,7 +432,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
data_path = new_data_path
|
||||
logger.info(f"Job {task_id}: Moved data file to {data_path}")
|
||||
|
||||
# Move config file to correct uploads directory (if it's a custom config)
|
||||
# 移动配置文件到正确的上传目录（如果是自定义配置）
|
||||
if config_path.parent != job_upload_dir and config_path.parent != Config.BASE_DIR:
|
||||
new_config_path = job_upload_dir / config_path.name
|
||||
if config_path != new_config_path:
|
||||
@ -451,7 +451,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "配置已加载，开始预处理...")
|
||||
|
||||
# 2. Data Preprocessing (files are already in correct directories)
|
||||
# 2. 数据预处理（文件已在正确目录中）
|
||||
logger.info(f"Job {task_id}: Starting preprocessing phase...")
|
||||
preprocess_start = time.time()
|
||||
|
||||
@ -465,7 +465,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "预处理完成，开始GasFlux分析...")
|
||||
|
||||
# Write modified config to a temp file
|
||||
# 将修改后的配置写入临时文件
|
||||
final_config_path = data_path.parent / "final_config.yaml"
|
||||
try:
|
||||
with open(final_config_path, 'w') as f:
|
||||
@ -480,7 +480,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "配置已加载，开始GasFlux分析...")
|
||||
|
||||
# 3. GasFlux Processing
|
||||
# 3. GasFlux 处理
|
||||
logger.info(f"Job {task_id}: Starting GasFlux analysis...")
|
||||
analysis_start = time.time()
|
||||
|
||||
@ -489,7 +489,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
analysis_duration = time.time() - analysis_start
|
||||
logger.info(f"Job {task_id}: GasFlux analysis completed in {analysis_duration:.3f}s")
|
||||
|
||||
# 提取krig_params数据（只保存关键数值）
|
||||
# 提取 krig_params 数据（只保存关键数值）
|
||||
krig_params_data = []
|
||||
if hasattr(processor, 'output_vars') and 'krig_parameters' in processor.output_vars:
|
||||
for gas, params in processor.output_vars['krig_parameters'].items():
|
||||
@ -499,12 +499,12 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
if isinstance(value, (int, float)):
|
||||
clean_params[key] = value
|
||||
elif hasattr(value, 'item') and hasattr(value, 'size'):
|
||||
# numpy数组：只处理单元素数组
|
||||
# numpy 数组：只处理单元素数组
|
||||
if value.size == 1:
|
||||
clean_params[key] = value.item()
|
||||
# 多元素数组跳过，不保存
|
||||
elif hasattr(value, 'item'):
|
||||
# 其他numpy对象尝试转换
|
||||
# 其他 numpy 对象尝试转换
|
||||
try:
|
||||
clean_params[key] = value.item()
|
||||
except ValueError:
|
||||
@ -518,12 +518,12 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "GasFlux分析完成，正在生成报告...")
|
||||
|
||||
# Collect results and generate full URLs
|
||||
# 收集结果并生成完整 URL
|
||||
logger.info(f"Job {task_id}: Collecting generated files from {job_output_dir}")
|
||||
results_start = time.time()
|
||||
results = []
|
||||
|
||||
# 先添加krig_params数据
|
||||
# 先添加 krig_params 数据
|
||||
results.extend(krig_params_data)
|
||||
|
||||
try:
|
||||
@ -534,7 +534,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"download_url": f"/download/{rel_path}", # 客户端可使用的相对 URL
|
||||
"size": file_size
|
||||
})
|
||||
logger.debug(f"Job {task_id}: Found output file: {f.name} ({file_size} bytes)")
|
||||
@ -552,7 +552,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
total_duration = time.time() - start_time
|
||||
logger.info(f"Job {task_id}: Processing complete. Total duration: {total_duration:.3f}s, {len(results)} files generated.")
|
||||
|
||||
# Record task completion time for statistics
|
||||
# 为统计记录任务完成时间
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_COMPLETED, "处理成功完成", results=results)
|
||||
@ -561,11 +561,11 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
total_duration = time.time() - start_time
|
||||
logger.error(f"Job {task_id}: Processing failed after {total_duration:.3f}s - Error: {str(e)}", exc_info=True)
|
||||
|
||||
# Record failed task processing time for statistics
|
||||
# 为统计记录失败任务处理时间
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
logger.error(f"Job {task_id}: Failed task details - Data: {data_path}, Config: {config_path}, Output: {job_output_dir}")
|
||||
|
||||
# Try to capture any partial results
|
||||
# 尝试捕获任何部分结果
|
||||
partial_results = []
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
@ -574,7 +574,7 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
partial_results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"download_url": f"/download/{rel_path}", # 客户端可使用的相对 URL
|
||||
"size": f.stat().st_size,
|
||||
"note": "partial_result"
|
||||
})
|
||||
@ -587,12 +587,12 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_FAILED, error=error_msg, results=partial_results if partial_results else None)
|
||||
|
||||
# Import GasFlux modules
|
||||
# 导入 GasFlux 模块
|
||||
logger.info("Importing GasFlux modules...")
|
||||
import_start = time.time()
|
||||
|
||||
try:
|
||||
# Try absolute imports first (more reliable)
|
||||
# 首先尝试绝对导入（更可靠）
|
||||
from src.gasflux.processing_pipelines import process_main
|
||||
from src.gasflux.data_processor import process_file
|
||||
from src.gasflux.reporting import generate_reports
|
||||
@ -612,62 +612,62 @@ except ImportError as e1:
|
||||
raise ImportError(f"Cannot import GasFlux modules: {e2}")
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app) # Initialize CORS
|
||||
CORS(app) # 初始化 CORS
|
||||
|
||||
# Enhanced logging configuration after app initialization
|
||||
# 应用初始化后的增强日志配置
|
||||
try:
|
||||
log_file_path = Path(Config.LOG_FILE)
|
||||
log_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create file handler
|
||||
# 创建文件处理器
|
||||
file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
|
||||
file_handler.setLevel(Config.get_log_level())
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
# Add file handler to logger
|
||||
# 将文件处理器添加到日志记录器
|
||||
logger.addHandler(file_handler)
|
||||
logger.info(f"File logging initialized. Log file: {log_file_path.absolute()}")
|
||||
print(f"Log file: {log_file_path.absolute()}") # Also print to console
|
||||
print(f"Log file: {log_file_path.absolute()}") # 同时输出到控制台
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to initialize file logging: {e}")
|
||||
logger.warning(f"Failed to initialize file logging: {e}")
|
||||
|
||||
logger.info("Flask application initialized")
|
||||
|
||||
# Request logging middleware
|
||||
# 请求日志中间件
|
||||
@app.before_request
|
||||
def log_request_info():
|
||||
"""Log incoming request details."""
|
||||
"""记录传入请求详情"""
|
||||
g.start_time = time.time()
|
||||
logger.info(f"REQUEST: {request.method} {request.url} - IP: {request.remote_addr} - User-Agent: {request.headers.get('User-Agent', 'Unknown')}")
|
||||
|
||||
@app.after_request
|
||||
def log_response_info(response):
|
||||
"""Log response details."""
|
||||
"""记录响应详情"""
|
||||
duration = time.time() - g.start_time
|
||||
logger.info(f"RESPONSE: {request.method} {request.url} - Status: {response.status_code} - Duration: {duration:.3f}s")
|
||||
|
||||
# Record statistics
|
||||
# 记录统计信息
|
||||
endpoint = request.url_rule.rule if request.url_rule else request.path
|
||||
stats_collector.record_request(request.method, endpoint, response.status_code, duration)
|
||||
|
||||
return response
|
||||
|
||||
# Initialize configuration from environment variables
|
||||
# 从环境变量初始化配置
|
||||
Config.init_base_dir()
|
||||
|
||||
# Apply configuration to app (directories will be created dynamically based on config)
|
||||
# ALLOWED_DATA_EXTENSIONS and ALLOWED_CONFIG_EXTENSIONS moved to shared.py
|
||||
# 将配置应用到应用（目录将根据配置动态创建）
|
||||
# ALLOWED_DATA_EXTENSIONS 和 ALLOWED_CONFIG_EXTENSIONS 已移至 shared.py
|
||||
|
||||
app.config['MAX_CONTENT_LENGTH'] = Config.MAX_CONTENT_LENGTH
|
||||
app.config['BASE_URL'] = Config.BASE_URL
|
||||
|
||||
# Set upload and output folders from config
|
||||
# 从配置设置上传和输出文件夹
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
|
||||
# Database and persistence configuration
|
||||
# 数据库和持久化配置
|
||||
if Config.DB_PATH:
|
||||
app.config['DB_PATH'] = Config.DB_PATH
|
||||
if Config.TASK_PERSIST_BACKEND:
|
||||
@ -676,27 +676,27 @@ app.config['JANITOR_DRY_RUN'] = str(Config.JANITOR_DRY_RUN).lower()
|
||||
if Config.ADMIN_BOOTSTRAP_KEY:
|
||||
app.config['ADMIN_BOOTSTRAP_KEY'] = Config.ADMIN_BOOTSTRAP_KEY
|
||||
|
||||
# Task cleanup configuration
|
||||
# 任务清理配置
|
||||
app.config['SUCCESSFUL_TASK_CLEANUP_AGE'] = Config.SUCCESSFUL_TASK_CLEANUP_AGE
|
||||
app.config['FAILED_TASK_CLEANUP_AGE'] = Config.FAILED_TASK_CLEANUP_AGE
|
||||
app.config['TASK_CLEANUP_INTERVAL'] = Config.TASK_CLEANUP_INTERVAL
|
||||
|
||||
# Debug logging for cleanup configuration
|
||||
# 清理配置的调试日志
|
||||
logger.info(f"App config: FAILED_TASK_CLEANUP_AGE = {app.config['FAILED_TASK_CLEANUP_AGE']}")
|
||||
logger.info(f"App config: TASK_CLEANUP_INTERVAL = {app.config['TASK_CLEANUP_INTERVAL']}")
|
||||
|
||||
# Log current configuration
|
||||
# 记录当前配置
|
||||
logger.info(f"Upload folder: {Config.UPLOAD_FOLDER}")
|
||||
logger.info(f"Output folder: {Config.OUTPUT_FOLDER}")
|
||||
logger.info(f"Configuration: {Config.to_dict()}")
|
||||
|
||||
# Ensure directories exist at startup
|
||||
# 确保启动时目录存在
|
||||
def setup_directories():
|
||||
logger.info("Initializing application directories...")
|
||||
logger.info("正在初始化应用目录...")
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Check if directories already exist
|
||||
# 检查目录是否已存在
|
||||
upload_exists = Config.UPLOAD_FOLDER.exists()
|
||||
output_exists = Config.OUTPUT_FOLDER.exists()
|
||||
|
||||
@ -706,26 +706,26 @@ def setup_directories():
|
||||
duration = time.time() - start_time
|
||||
logger.info(f"Directories initialized in {duration:.3f}s: {Config.UPLOAD_FOLDER} ({'existing' if upload_exists else 'created'}), {Config.OUTPUT_FOLDER} ({'existing' if output_exists else 'created'})")
|
||||
|
||||
# Log directory permissions
|
||||
# 记录目录权限
|
||||
upload_writable = os.access(Config.UPLOAD_FOLDER, os.W_OK)
|
||||
output_writable = os.access(Config.OUTPUT_FOLDER, os.W_OK)
|
||||
logger.info(f"Directory permissions - Upload writable: {upload_writable}, Output writable: {output_writable}")
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"Failed to create directories after {duration:.3f}s: {e}")
|
||||
logger.error(f"创建目录失败，耗时 {duration:.3f}s: {e}")
|
||||
raise
|
||||
|
||||
# setup_directories() - commented out to avoid creating directories at startup
|
||||
# Directories will be created dynamically based on config when processing tasks
|
||||
# setup_directories() - 已注释掉以避免在启动时创建目录
|
||||
# 目录将在处理任务时根据配置动态创建
|
||||
|
||||
# allowed_file moved to shared.py
|
||||
# allowed_file 已移至 shared.py
|
||||
|
||||
# Initialize database and start background services
|
||||
# 初始化数据库并启动后台服务
|
||||
from .db import init_app as init_db
|
||||
init_db(app)
|
||||
|
||||
# Import blueprints after app initialization to avoid circular imports
|
||||
# 在应用初始化后导入蓝图以避免循环导入
|
||||
from .blueprints.health import health_bp
|
||||
from .blueprints.upload import upload_bp
|
||||
from .blueprints.tasks import tasks_bp
|
||||
@ -737,7 +737,7 @@ from .blueprints.download import download_bp
|
||||
from .blueprints.web import web_bp
|
||||
from .blueprints.api_keys import api_keys_bp
|
||||
|
||||
# Register blueprints
|
||||
# 注册蓝图
|
||||
app.register_blueprint(health_bp)
|
||||
app.register_blueprint(upload_bp)
|
||||
app.register_blueprint(tasks_bp)
|
||||
@ -749,20 +749,20 @@ app.register_blueprint(download_bp)
|
||||
app.register_blueprint(web_bp)
|
||||
app.register_blueprint(api_keys_bp)
|
||||
|
||||
# Task status persistence now uses SQLite only
|
||||
# JSON persistence has been disabled - functions removed from shared.py
|
||||
# 任务状态持久化现在仅使用 SQLite
|
||||
# JSON 持久化已禁用 - 函数已从 shared.py 中移除
|
||||
|
||||
# Initialize janitor for background cleanup
|
||||
# 初始化清理工具进行后台清理
|
||||
try:
|
||||
from .janitor import start_janitor, reconcile_tasks_on_startup
|
||||
with app.app_context():
|
||||
reconcile_tasks_on_startup()
|
||||
# No longer need to load task status into memory
|
||||
# 不再需要将任务状态加载到内存
|
||||
start_janitor(app)
|
||||
except Exception as e:
|
||||
print(f"⚠ Failed to setup task persistence: {e}")
|
||||
print(f"⚠ 设置任务持久化失败: {e}")
|
||||
|
||||
# _get_file_type and _format_response moved to shared.py
|
||||
# _get_file_type 和 _format_response 已移至 shared.py
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pybaselines as pybs
|
||||
from . import plotting
|
||||
|
||||
# 自定义阈值函数，避免依赖scikit-image
|
||||
def custom_threshold(data):
|
||||
@ -52,11 +51,12 @@ def algorithmic_baseline(
|
||||
background = (df[gas] - bkg)[bkg_points]
|
||||
signal = (df[gas] - bkg)[~bkg_points]
|
||||
df[f"{gas}_signal"] = np.invert(bkg_points)
|
||||
fig = plotting.background_plotting(df, gas)
|
||||
|
||||
fig = None # plotting disabled
|
||||
output_text = (
|
||||
f"Baseline algorithm: {algorithm}\n"
|
||||
f"Positive and negative 95% percentile of baseline: {np.percentile(background, 2.5):.2f} ppm, \
|
||||
{np.percentile(background, 97.5):.2f} ppm\n"
|
||||
f"Positive and negative 95% percentile of baseline: {np.percentile(background, 2.5):.2f} ppm, "
|
||||
f"{np.percentile(background, 97.5):.2f} ppm\n"
|
||||
f"Mean of baseline: {np.mean(background):.2f} ppm\n"
|
||||
f"Minimum and maximum of baseline: {np.min(background):.2f} ppm, {np.max(background):.2f} ppm\n"
|
||||
f"Signal points: {len(signal)}; background points: {len(background)}\n"
|
||||
|
||||
@ -2,11 +2,24 @@
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import skgstat as skg
|
||||
from scipy import integrate
|
||||
import os
|
||||
|
||||
from . import plotting
|
||||
|
||||
# Set matplotlib backend before importing anything that might use it
|
||||
os.environ['MPLBACKEND'] = 'Agg'
|
||||
|
||||
# Import matplotlib and set backend explicitly
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Import scikit-gstat
|
||||
import skgstat as skg
|
||||
|
||||
|
||||
def simpsonintegrate(array: np.ndarray, x_cell_size: float, y_cell_size: float) -> float:
|
||||
"""Function to obtain the volume of the krig in kgh⁻¹, i.e. the cut-fill volume
|
||||
@ -42,7 +55,6 @@ def ordinary_kriging(
|
||||
):
|
||||
"""Function to calculate the ordinary kriging of a gas in a dataframe, after calculating a semivariogram."""
|
||||
gasflux = f"{gas}_kg_h_m2"
|
||||
skg.plotting.backend("plotly") # type: ignore
|
||||
cut_ground = ordinary_kriging_settings["cut_ground"]
|
||||
semivariogram = directional_gas_semivariogram(df, x, y, gasflux, semivariogram_filter, **semivariogram_settings)
|
||||
ok = skg.OrdinaryKriging(
|
||||
@ -87,8 +99,11 @@ def ordinary_kriging(
|
||||
# np.nan_to_num(error_1s, copy=False, nan=0)
|
||||
volume_error = simpsonintegrate(error_1s, x_cell_size, y_cell_size)
|
||||
|
||||
contour_plot = plotting.contour_krig(df=df, gas=gas, xx=xx, yy=yy, field=field, x=x, y=y, cut_ground=cut_ground)
|
||||
grid_plot = plotting.heatmap_krig(xx, yy, field)
|
||||
# Plots disabled
|
||||
contour_plot = None
|
||||
grid_plot = None
|
||||
semivariogram_plot = None
|
||||
|
||||
output_text = (
|
||||
f"The emissions flux of {gas.upper()} is {volume:.3f}kgh⁻¹; "
|
||||
f"the cut and fill volumes of the grid are {volumepos:.3f} and {volumeneg:.3f}kgh⁻¹. "
|
||||
@ -107,7 +122,6 @@ def ordinary_kriging(
|
||||
"error field (1 sigma)": error_1s,
|
||||
"volume_error": volume_error,
|
||||
}
|
||||
semivariogram_plot = semivariogram.plot(show=False)
|
||||
|
||||
return krig_variables, output_text, contour_plot, grid_plot, semivariogram_plot
|
||||
|
||||
|
||||
@ -56,6 +56,6 @@ def make_prediction(
|
||||
predictions = model.predict(df[cols_for_model])
|
||||
|
||||
df["predictions"] = predictions
|
||||
fig = plotting.scatter_3d(df)
|
||||
fig = None # plotting disabled
|
||||
|
||||
return df, fig
|
||||
|
||||
@ -1,609 +1,44 @@
|
||||
"""Various plotting functions mainly based around plotly."""
|
||||
|
||||
import matplotlib.colors as mcolors
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import plotly.io as pio
|
||||
import simplekml
|
||||
from plotly.subplots import make_subplots
|
||||
|
||||
from . import processing
|
||||
|
||||
pio.templates["default"] = go.layout.Template(
|
||||
layout=go.Layout(
|
||||
margin=go.layout.Margin(l=0, r=0, b=0, t=0, pad=0),
|
||||
),
|
||||
)
|
||||
|
||||
pio.templates.default = "simple_white+default"
|
||||
|
||||
|
||||
styling = {
|
||||
"colorscale": "geyser",
|
||||
}
|
||||
"""
|
||||
Lightweight stub plotting module to disable heavy visualization dependencies.
|
||||
All functions return None so callers can safely check for truthiness.
|
||||
"""
|
||||
|
||||
|
||||
def blank_figure():
|
||||
fig = go.Figure()
|
||||
return fig
|
||||
return None
|
||||
|
||||
|
||||
def scatter_3d(
|
||||
df: pd.DataFrame,
|
||||
color: str = "",
|
||||
colorbar_title: str = "",
|
||||
timestamp: str = "timestamp",
|
||||
x: str = "utm_easting",
|
||||
y: str = "utm_northing",
|
||||
z: str = "height_ato",
|
||||
courses: bool = False,
|
||||
):
|
||||
fig = px.scatter_3d(df, x=x, y=y, z=z)
|
||||
|
||||
if color:
|
||||
custom_data = [df[timestamp]]
|
||||
if courses:
|
||||
custom_data.extend([df["course_elevation"], df["course_azimuth"]])
|
||||
custom_data = np.stack(custom_data, axis=-1)
|
||||
hover_template = [
|
||||
f"{x}: %{{x:.2f}}",
|
||||
f"{y}: %{{y:.2f}}",
|
||||
f"{z}: %{{z:.2f}}",
|
||||
f"{color}: %{{marker.color:.2f}}",
|
||||
f"{timestamp}: %{{customdata[0]|%Y-%m-%d %H:%M:%S}}",
|
||||
"Index: %{pointNumber}",
|
||||
]
|
||||
|
||||
if courses:
|
||||
hover_template.extend(
|
||||
[
|
||||
"Course Elevation: %{customdata[1]:.2f}",
|
||||
"Course Azimuth: %{customdata[2]:.2f}",
|
||||
]
|
||||
)
|
||||
hover_template_str = "<br>".join(hover_template)
|
||||
fig.update_traces(
|
||||
marker=dict(
|
||||
color=df[color],
|
||||
size=4,
|
||||
opacity=0.5,
|
||||
colorscale=styling["colorscale"],
|
||||
colorbar=dict(title=colorbar_title),
|
||||
),
|
||||
customdata=custom_data,
|
||||
hovertemplate=hover_template_str,
|
||||
)
|
||||
|
||||
return fig
|
||||
def scatter_3d(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def scatter_2d(
|
||||
df: pd.DataFrame,
|
||||
x: str,
|
||||
color: str,
|
||||
y: str = "height_ato",
|
||||
**kwargs,
|
||||
):
|
||||
fig = px.scatter(
|
||||
df,
|
||||
x=x,
|
||||
y=y,
|
||||
color=color,
|
||||
color_continuous_scale=styling["colorscale"],
|
||||
opacity=0.8,
|
||||
**kwargs,
|
||||
)
|
||||
fig.update_traces(
|
||||
customdata=df.index,
|
||||
hovertemplate="<br>".join(
|
||||
[
|
||||
"x: %{x:.2f}",
|
||||
"height_ato: %{y:.2f}",
|
||||
f"{color}: %{{marker.color:.2f}}",
|
||||
"Time: %{customdata}",
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
return fig
|
||||
def scatter_2d(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def time_series(
|
||||
df: pd.DataFrame,
|
||||
ys: str | list[str],
|
||||
x: str = "timestamp",
|
||||
color: str | None = None,
|
||||
split=None,
|
||||
y_mins: float | list[float | int] | None = None,
|
||||
rolling_average: bool = True,
|
||||
scatter: bool = True,
|
||||
rolling_window: int = 5,
|
||||
y_titles: str | list[str] | None = None,
|
||||
legend: bool = True,
|
||||
) -> go.Figure:
|
||||
colors = px.colors.qualitative.Plotly
|
||||
|
||||
if isinstance(ys, str):
|
||||
ys = [ys]
|
||||
if y_titles is None:
|
||||
y_titles = ys
|
||||
single_title = False
|
||||
elif isinstance(y_titles, str):
|
||||
y_titles = [y_titles]
|
||||
single_title = True
|
||||
elif isinstance(y_titles, list):
|
||||
if len(y_titles) != len(ys):
|
||||
raise ValueError("Length of y_titles must be equal to length of ys")
|
||||
single_title = False
|
||||
else:
|
||||
raise ValueError("Invalid y_titles value")
|
||||
if isinstance(y_mins, (float | int)):
|
||||
y_mins = [y_mins]
|
||||
if isinstance(y_mins, list):
|
||||
if len(y_mins) != len(ys):
|
||||
raise ValueError("Length of y_mins must be equal to length of ys")
|
||||
|
||||
fig = go.Figure()
|
||||
|
||||
axis_space = 0.05
|
||||
domain_start = axis_space * (len(ys)) if len(ys) > 1 else 0
|
||||
fig.update_layout(
|
||||
xaxis=dict(
|
||||
domain=[domain_start, 1],
|
||||
),
|
||||
)
|
||||
|
||||
for i, y in enumerate(ys):
|
||||
yaxis_name = f"yaxis{i+1}"
|
||||
yaxis_ref = f"y{i+1}"
|
||||
|
||||
trace_color = "black" if single_title and i == 0 else colors[i % len(colors)]
|
||||
|
||||
marker_i = dict(size=8, opacity=0.3 if rolling_average else 0.5, color=trace_color)
|
||||
if color is not None:
|
||||
marker_i["color"] = df[color] # type: ignore
|
||||
marker_i["colorscale"] = styling["colorscale"]
|
||||
|
||||
hover_template = f"{x}: %{{x}}<br>{y}: %{{y:.2f}}<br>"
|
||||
if color:
|
||||
hover_template += f"{color}: %{{marker.color:.2f}}<br>"
|
||||
|
||||
if scatter:
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[y],
|
||||
name=y,
|
||||
mode="markers",
|
||||
marker=marker_i,
|
||||
yaxis=yaxis_ref,
|
||||
hovertemplate=hover_template,
|
||||
showlegend=legend,
|
||||
)
|
||||
)
|
||||
|
||||
if rolling_average:
|
||||
df[f"rolling_avg_{i}"] = df[y].rolling(window=rolling_window, min_periods=1).mean()
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[f"rolling_avg_{i}"],
|
||||
name=f"{y} {rolling_window}-point avg",
|
||||
mode="lines",
|
||||
line=dict(color=trace_color, width=2),
|
||||
yaxis=yaxis_ref,
|
||||
showlegend=legend,
|
||||
)
|
||||
)
|
||||
|
||||
y_data = df[y]
|
||||
y_min_var = y_data.min()
|
||||
y_max_var = y_data.max()
|
||||
y_range = y_max_var - y_min_var or y_max_var * 0.05
|
||||
|
||||
y_axis_min = y_mins[i] if y_mins is not None and y_mins[i] is not None else y_min_var - y_range * 0.05
|
||||
y_axis_max = y_max_var + y_range * 0.05
|
||||
|
||||
if single_title and i == 0:
|
||||
axis_title = dict(text=y_titles[0], font=dict(color="black"))
|
||||
elif not single_title:
|
||||
axis_title = dict(text=y_titles[i], font=dict(color=trace_color))
|
||||
else:
|
||||
axis_title = None
|
||||
|
||||
axis_config = dict(
|
||||
title=axis_title,
|
||||
tickfont=dict(color=trace_color),
|
||||
range=[y_axis_min, y_axis_max],
|
||||
side="left",
|
||||
position=axis_space * i if i > 0 else None,
|
||||
anchor="free" if i > 0 else None,
|
||||
overlaying="y" if i > 0 else None,
|
||||
showgrid=(i == 0),
|
||||
)
|
||||
|
||||
fig.layout[yaxis_name] = axis_config
|
||||
|
||||
if split is not None:
|
||||
fig.add_shape(
|
||||
type="line",
|
||||
xref="x",
|
||||
yref="paper",
|
||||
x0=split,
|
||||
y0=0,
|
||||
x1=split,
|
||||
y1=1,
|
||||
line=dict(color="red", width=2),
|
||||
)
|
||||
|
||||
return fig
|
||||
def time_series(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def background_plotting(df: pd.DataFrame, gas: str):
|
||||
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
||||
ymin = df[gas].min()
|
||||
ymax = df[gas].max()
|
||||
ylim = [ymin * 0.95, ymax * 1.05]
|
||||
y2min = df[f"{gas}_normalised"].min()
|
||||
y2lim = (y2min, y2min + (ylim[1] - ylim[0]))
|
||||
fig.update_yaxes(range=ylim, secondary_y=False, title_text=f"Sensor {gas} (ppm)")
|
||||
fig.update_yaxes(range=y2lim, secondary_y=True, title_text=f"Normalised {gas} (ppm)")
|
||||
fig.add_scatter(x=df["timestamp"], y=df[gas], opacity=0.3, name="Raw Data")
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"], y=df[f"{gas}_fit"], mode="lines", name="Fitted Background", line=dict(dash="dash")
|
||||
)
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"], y=df[f"{gas}_normalised"], yaxis="y2", name="Normalised Data", mode="lines", opacity=0.5
|
||||
)
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"],
|
||||
y=np.where(df[f"{gas}_signal"], df[f"{gas}_normalised"], np.nan),
|
||||
yaxis="y2",
|
||||
name="Classed as signal",
|
||||
mode="lines",
|
||||
opacity=0.5,
|
||||
# color
|
||||
# mode="markers",
|
||||
# marker=dict(size=3),
|
||||
)
|
||||
|
||||
return fig
|
||||
def background_plotting(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def windrose_process(df: pd.DataFrame):
|
||||
beaufort = {
|
||||
"0": [0, 1],
|
||||
"1": [1, 2],
|
||||
"2": [2, 4],
|
||||
"3": [4, 6],
|
||||
"4": [6, 9],
|
||||
"5": [9, 11],
|
||||
"6": [11, 14],
|
||||
"7": [14, 17],
|
||||
"8": [17, 21],
|
||||
"9": [21, 25],
|
||||
"10": [25, 29],
|
||||
"11": [29, 33],
|
||||
"12": [33, 200],
|
||||
}
|
||||
|
||||
beaufort_ms = {
|
||||
"0": "0-1",
|
||||
"1": "1-2",
|
||||
"2": "2-4",
|
||||
"3": "4-6",
|
||||
"4": "6-9",
|
||||
"5": "9-11",
|
||||
"6": "11-14",
|
||||
"7": "14-17",
|
||||
"8": "17-21",
|
||||
"9": "21-25",
|
||||
"10": "25-29",
|
||||
"11": "29-33",
|
||||
"12": "33+",
|
||||
}
|
||||
|
||||
cardinals = {
|
||||
"N1": [0, 11.25],
|
||||
"NNE": [11.25, 33.75],
|
||||
"NE": [33.75, 56.25],
|
||||
"ENE": [56.25, 78.75],
|
||||
"E": [78.75, 101.25],
|
||||
"ESE": [101.25, 123.75],
|
||||
"SE": [123.75, 146.25],
|
||||
"SSE": [146.25, 168.75],
|
||||
"S": [168.75, 191.25],
|
||||
"SSW": [191.25, 213.75],
|
||||
"SW": [213.75, 236.25],
|
||||
"WSW": [236.25, 258.75],
|
||||
"W": [258.75, 281.25],
|
||||
"WNW": [281.25, 303.75],
|
||||
"NW": [303.75, 326.25],
|
||||
"NNW": [326.25, 348.75],
|
||||
"N2": [348.75, 360],
|
||||
}
|
||||
df["wind_direction_bin"] = pd.cut(
|
||||
df["winddir"],
|
||||
bins=[lower for lower, upper in cardinals.values()] + [list(cardinals.values())[-1][1]],
|
||||
labels=[key for key in cardinals],
|
||||
right=False,
|
||||
)
|
||||
|
||||
df["wind_direction_bin"] = (
|
||||
df["wind_direction_bin"].map(lambda x: "N" if x in ["N1", "N2"] else x).astype("category")
|
||||
)
|
||||
df["beaufort"] = pd.cut(
|
||||
df["windspeed"],
|
||||
bins=[lower for lower, upper in beaufort.values()] + [list(beaufort.values())[-1][1]],
|
||||
labels=[key for key in beaufort],
|
||||
right=False,
|
||||
)
|
||||
df["beaufort_ms"] = df["beaufort"].map(beaufort_ms)
|
||||
df_windrose = df.groupby(["wind_direction_bin", "beaufort"], observed=False).size().reset_index(name="count") # type: ignore
|
||||
df_windrose["frequency"] = df_windrose["count"] / df_windrose["count"].sum() * 100
|
||||
df_windrose["wind_direction_bin_degs"] = df_windrose["wind_direction_bin"].cat.rename_categories(
|
||||
{
|
||||
"N": 0,
|
||||
"NNE": 22.5,
|
||||
"NE": 45,
|
||||
"ENE": 67.5,
|
||||
"E": 90,
|
||||
"ESE": 112.5,
|
||||
"SE": 135,
|
||||
"SSE": 157.5,
|
||||
"S": 180,
|
||||
"SSW": 202.5,
|
||||
"SW": 225,
|
||||
"WSW": 247.5,
|
||||
"W": 270,
|
||||
"WNW": 292.5,
|
||||
"NW": 315,
|
||||
"NNW": 337.5,
|
||||
},
|
||||
)
|
||||
df_windrose["beaufort"] = df_windrose["beaufort"].astype(int)
|
||||
return df_windrose
|
||||
def windrose(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def windrose_graph(df, plot_transect=False, theta1=None, theta2=None):
|
||||
n_colors = 13
|
||||
colors = px.colors.sample_colorscale("turbo", [n / (n_colors - 1) for n in range(n_colors)])
|
||||
fig = px.bar_polar(
|
||||
df,
|
||||
r="frequency",
|
||||
theta="wind_direction_bin_degs",
|
||||
color="beaufort",
|
||||
labels={
|
||||
"frequency": "Frequency (%)",
|
||||
"wind_direction_bin": "Direction",
|
||||
"beaufort": "Beaufort Scale",
|
||||
},
|
||||
color_discrete_map=colors,
|
||||
)
|
||||
fig.update_layout(polar=dict(radialaxis={"visible": False, "showticklabels": False}))
|
||||
fig.update_layout(
|
||||
polar=dict(
|
||||
angularaxis={
|
||||
"showgrid": False,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
fig.update_layout(polar_bargap=0)
|
||||
if plot_transect:
|
||||
max_freq = df.groupby("wind_direction_bin", observed=False)["frequency"].sum().max()
|
||||
fig.add_trace(
|
||||
go.Scatterpolar(
|
||||
r=[max_freq, max_freq],
|
||||
theta=[theta1, theta2],
|
||||
mode="lines",
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
showlegend=False,
|
||||
),
|
||||
)
|
||||
return fig
|
||||
def outliers(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def windrose(df: pd.DataFrame, plot_transect=False):
|
||||
df_windrose = windrose_process(df)
|
||||
if plot_transect:
|
||||
theta1, theta2 = processing.bimodal_azimuth(df)
|
||||
fig = windrose_graph(df_windrose, plot_transect=plot_transect, theta1=theta1, theta2=theta2)
|
||||
else:
|
||||
fig = windrose_graph(df_windrose, plot_transect=plot_transect)
|
||||
return fig
|
||||
def contour_krig(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def outliers(original_data: pd.Series, fence_high: float, fence_low: float):
|
||||
outliers = np.array(original_data > fence_high) | (original_data < fence_low)
|
||||
|
||||
fig = make_subplots(rows=1, cols=2, shared_yaxes=True)
|
||||
fig.add_trace(px.strip(original_data, color=outliers).data[0], row=1, col=1)
|
||||
if sum(outliers) > 0:
|
||||
fig.add_trace(px.strip(original_data, color=outliers).data[1], row=1, col=1)
|
||||
fig.add_shape(
|
||||
go.layout.Shape(
|
||||
type="line",
|
||||
x0=-0.5,
|
||||
y0=fence_high,
|
||||
x1=0.5,
|
||||
y1=fence_high,
|
||||
line=dict(color="red", width=2),
|
||||
),
|
||||
row=1,
|
||||
col=1,
|
||||
)
|
||||
fig.add_shape(
|
||||
go.layout.Shape(
|
||||
type="line",
|
||||
x0=-0.5,
|
||||
y0=fence_low,
|
||||
x1=0.5,
|
||||
y1=fence_low,
|
||||
line=dict(color="red", width=2),
|
||||
),
|
||||
row=1,
|
||||
col=1,
|
||||
)
|
||||
fig.update_traces(offsetgroup=0)
|
||||
|
||||
fig.add_trace(px.scatter(original_data, color=outliers).data[0], row=1, col=2)
|
||||
if sum(outliers) > 0:
|
||||
fig.add_trace(px.scatter(original_data, color=outliers).data[1], row=1, col=2)
|
||||
fig.update_layout(showlegend=False, yaxis_title="Windspeed (ms⁻¹)")
|
||||
|
||||
return fig
|
||||
def heatmap_krig(*args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def contour_krig(
|
||||
df: pd.DataFrame,
|
||||
gas: str,
|
||||
# array of float 64
|
||||
xx: np.ndarray,
|
||||
yy: np.ndarray,
|
||||
field: np.ndarray,
|
||||
cut_ground: bool = False,
|
||||
x: str = "x",
|
||||
y: str = "height_ato",
|
||||
) -> go.Figure:
|
||||
if np.isnan(field).all():
|
||||
return blank_figure()
|
||||
fig = go.Figure()
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[y],
|
||||
mode="markers",
|
||||
marker={
|
||||
"color": df[f"{gas}_normalised"],
|
||||
"colorscale": styling["colorscale"],
|
||||
"showscale": True,
|
||||
"colorbar": {
|
||||
"title": f"{gas} (ppm)",
|
||||
},
|
||||
},
|
||||
showlegend=False,
|
||||
)
|
||||
)
|
||||
fig.add_trace(
|
||||
go.Contour(
|
||||
z=field.T,
|
||||
x=xx[:, 0],
|
||||
y=yy[0, :],
|
||||
contours={
|
||||
"start": field.min(),
|
||||
"end": field.max(),
|
||||
"size": (field[~np.isnan(field)].max() - field[~np.isnan(field)].min()) / 21,
|
||||
},
|
||||
colorscale=styling["colorscale"],
|
||||
opacity=0.5,
|
||||
showlegend=False,
|
||||
showscale=False,
|
||||
)
|
||||
)
|
||||
fig.update_xaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="horizontal distance on projected flux plane (m)",
|
||||
range=[np.min(xx), np.max(xx)],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=20,
|
||||
)
|
||||
fig.update_yaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="height above takeoff (m)",
|
||||
range=[np.min(yy), np.max(yy)],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=10,
|
||||
)
|
||||
if cut_ground:
|
||||
resolution = 200 # how many points to interpolate over
|
||||
df["ground_elevation_ato"] = df.loc[:, "height_ato"] - df.loc[:, "height_agl"]
|
||||
df_sorted = df.dropna(subset=[x, "ground_elevation_ato"]).sort_values(x)
|
||||
x_min, x_max = df_sorted[x].min(), df_sorted[x].max()
|
||||
x_interp = np.linspace(x_min, x_max, resolution)
|
||||
ground_ato_interp = np.interp(x_interp, df_sorted[x], df_sorted["ground_elevation_ato"])
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=x_interp,
|
||||
y=ground_ato_interp,
|
||||
mode="lines",
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
name="Interpolated Ground Level",
|
||||
)
|
||||
)
|
||||
fig.layout.coloraxis.colorbar.title = "Emissions flux (kg⋅m⁻²⋅h⁻¹)"
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def heatmap_krig(xx: np.ndarray, yy: np.ndarray, field: np.ndarray):
|
||||
fig = px.imshow(field.T, x=xx[:, 0], y=yy[0, :], color_continuous_scale=styling["colorscale"], origin="lower")
|
||||
fig.layout.coloraxis.colorbar.title = "Emissions flux (kg⋅m⁻²⋅h⁻¹)"
|
||||
fig.update_xaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="horizontal distance on cylindrical projected flux plane (m)",
|
||||
range=[xx.min(), xx.max()],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=20,
|
||||
)
|
||||
fig.update_yaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="height above ground level (m)",
|
||||
range=[yy.min(), yy.max()],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=10,
|
||||
)
|
||||
fig.update_layout(coloraxis_colorbar=dict(len=0.25))
|
||||
return fig
|
||||
|
||||
|
||||
def create_kml_file(data: pd.DataFrame, output_file: str, column: str, altitudemode: str):
|
||||
kml = simplekml.Kml()
|
||||
|
||||
min_value = data[column].min()
|
||||
max_value = data[column].max()
|
||||
|
||||
custom_colors = [
|
||||
"#008080",
|
||||
"#70a494",
|
||||
"#b4c8a8",
|
||||
"#f6edbd",
|
||||
"#edbb8a",
|
||||
"#de8a5a",
|
||||
"#ca562c",
|
||||
] # based on plotly geyser
|
||||
cmap = mcolors.LinearSegmentedColormap.from_list("custom_cmap", custom_colors)
|
||||
|
||||
for _index, row in data.iterrows():
|
||||
col_normalized = (row[column] - min) / (max_value - min_value)
|
||||
color = mcolors.rgb2hex(cmap(col_normalized))
|
||||
|
||||
pnt = kml.newpoint(coords=[(row["longitude"], row["latitude"], row["height_ato"])], altitudemode=altitudemode)
|
||||
pnt.iconstyle.icon.href = "http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png"
|
||||
pnt.iconstyle.color = simplekml.Color.rgb(int(color[1:3], 16), int(color[3:5], 16), int(color[5:], 16))
|
||||
pnt.iconstyle.scale = 0.6
|
||||
pnt.description = f"Concentration: {row[column]} ppm"
|
||||
|
||||
kml.save(output_file)
|
||||
def create_kml_file(*args, **kwargs):
|
||||
return None
|
||||
|
||||
@ -1,11 +1,9 @@
|
||||
"""Functions that organise the data into standard columns in pandas dataframes. Conversion functions (e.g. WGS84 to UTM)
|
||||
are here but transformations take place in processing.py"""
|
||||
|
||||
import geopandas as gpd
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from . import plotting
|
||||
from pyproj import Transformer
|
||||
from .processing import circ_median
|
||||
|
||||
|
||||
@ -37,19 +35,52 @@ def timestamp_from_four_columns(df):
|
||||
|
||||
# add UTM from latitudes and longitudes
|
||||
def add_utm(df: pd.DataFrame) -> pd.DataFrame:
|
||||
gdf = gpd.GeoDataFrame( # type: ignore
|
||||
df,
|
||||
geometry=gpd.points_from_xy(df["longitude"], df["latitude"], crs="EPSG:4326"),
|
||||
)
|
||||
utm = gdf.estimate_utm_crs()
|
||||
gdf = gdf.to_crs(utm)
|
||||
if not isinstance(gdf, gpd.GeoDataFrame):
|
||||
raise TypeError("Failed to reproject to a GeoDataFrame")
|
||||
gdf["utm_easting"] = gdf.geometry.x
|
||||
gdf["utm_northing"] = gdf.geometry.y
|
||||
output_df = pd.DataFrame(gdf.drop(columns="geometry"))
|
||||
"""
|
||||
Convert WGS84 coordinates to UTM using pyproj.
|
||||
|
||||
return output_df
|
||||
This function replaces the geopandas implementation with a lighter pyproj-based solution.
|
||||
"""
|
||||
# Make a copy to avoid modifying the original DataFrame
|
||||
df = df.copy()
|
||||
|
||||
# Get the average longitude to determine the UTM zone
|
||||
# For simplicity, we'll use the first valid longitude to determine the zone
|
||||
# In production, you might want to use the centroid or handle multiple zones
|
||||
valid_lons = df["longitude"].dropna()
|
||||
if len(valid_lons) == 0:
|
||||
raise ValueError("No valid longitude values found")
|
||||
|
||||
# Calculate UTM zone from longitude
|
||||
# UTM zones are 6 degrees wide, starting from -180
|
||||
lon = valid_lons.iloc[0] # Use first valid longitude
|
||||
zone_number = int((lon + 180) / 6) + 1
|
||||
|
||||
# Determine if it's northern or southern hemisphere
|
||||
# Use first valid latitude
|
||||
valid_lats = df["latitude"].dropna()
|
||||
if len(valid_lats) == 0:
|
||||
raise ValueError("No valid latitude values found")
|
||||
|
||||
lat = valid_lats.iloc[0]
|
||||
hemisphere = 'north' if lat >= 0 else 'south'
|
||||
|
||||
# Create UTM CRS string
|
||||
utm_crs = f"EPSG:326{zone_number:02d}" if hemisphere == 'north' else f"EPSG:327{zone_number:02d}"
|
||||
|
||||
# Create transformer from WGS84 to UTM
|
||||
transformer = Transformer.from_crs("EPSG:4326", utm_crs, always_xy=True)
|
||||
|
||||
# Transform coordinates
|
||||
utm_easting, utm_northing = transformer.transform(
|
||||
df["longitude"].values,
|
||||
df["latitude"].values
|
||||
)
|
||||
|
||||
# Add UTM coordinates to DataFrame
|
||||
df["utm_easting"] = utm_easting
|
||||
df["utm_northing"] = utm_northing
|
||||
|
||||
return df
|
||||
|
||||
|
||||
# add columns for drone course azimuth and elevation
|
||||
@ -97,7 +128,9 @@ def remove_outliers(df: pd.DataFrame, column: str, name: str):
|
||||
iqr = q3 - q1
|
||||
fence_low = q1 - 3 * iqr
|
||||
fence_high = q3 + 3 * iqr
|
||||
fig = plotting.outliers(df[column], fence_high, fence_low)
|
||||
|
||||
fig = None # plotting disabled
|
||||
|
||||
outliers = df.loc[(df[column] < fence_low) | (df[column] > fence_high)]
|
||||
if len(outliers) > 0:
|
||||
print(f"{len(outliers)} outliers removed from {name} {column} data")
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
"""Processing function, usually implying some kind of filtering or data transformation."""
|
||||
|
||||
from matplotlib.figure import Figure
|
||||
from itertools import groupby
|
||||
from typing import Any
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy import odr
|
||||
@ -134,16 +132,16 @@ def bimodal_elevation(
|
||||
return (mode, -mode)
|
||||
|
||||
|
||||
def height_transect_splitter(df: pd.DataFrame, height_col: str = "height_ato") -> tuple[pd.DataFrame, Figure]:
|
||||
def height_transect_splitter(df: pd.DataFrame, height_col: str = "height_ato") -> tuple[pd.DataFrame, None]:
|
||||
"""
|
||||
Splits the dataset into height-based transects and plots histogram peaks to identify prominent
|
||||
Splits the dataset into height-based transects using histogram peaks to identify prominent
|
||||
height ranges. Only works if the flights are flat.
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): The input dataframe containing height data.
|
||||
|
||||
Returns:
|
||||
tuple: Modified dataframe with transect labels and a figure showing the histogram with peaks.
|
||||
tuple: Modified dataframe with transect labels and None (plotting disabled).
|
||||
"""
|
||||
df = df.copy()
|
||||
heights = df[height_col].to_numpy()
|
||||
@ -158,12 +156,8 @@ def height_transect_splitter(df: pd.DataFrame, height_col: str = "height_ato") -
|
||||
transect_edges = (bin_centers[peaks][:-1] + bin_centers[peaks][1:]) / 2
|
||||
transect_edges = np.append(heights.min(), transect_edges)
|
||||
transect_edges = np.append(transect_edges, heights.max())
|
||||
fig, ax = plt.subplots()
|
||||
ax.stairs(edges=bin_edges, values=counts, fill=True)
|
||||
ax.plot(bin_centers[peaks], counts[peaks], "x", color="red")
|
||||
ax.vlines(transect_edges, ymin=0, ymax=max(counts), color="red")
|
||||
df["transect_num"] = pd.cut(df[height_col], bins=list(transect_edges), labels=False, include_lowest=True) # type: ignore
|
||||
return df, fig
|
||||
return df, None
|
||||
|
||||
|
||||
def add_transect_azimuth_switches(df: pd.DataFrame, threshold=150, shift=3) -> pd.DataFrame:
|
||||
|
||||
@ -4,7 +4,6 @@ from pathlib import Path
|
||||
from scipy import stats
|
||||
|
||||
import pandas as pd
|
||||
import plotly.graph_objects as go
|
||||
import yaml
|
||||
|
||||
from src.gasflux import background,plotting,processing,reporting,interpolation,pre_processing,gas
|
||||
@ -124,16 +123,9 @@ class InSituSensorStrategy(SensorStrategy):
|
||||
def process(self):
|
||||
logger.info("Processing in-situ (point) data")
|
||||
for gas in self.data_processor.gases:
|
||||
self.data_processor.figs["scatter_3d"][gas] = plotting.scatter_3d(
|
||||
df=self.data_processor.df, color=gas, colorbar_title=f"{gas.upper()} flux (kg/m²/h)"
|
||||
)
|
||||
if SpatialProcessingStrategy == CurtainSpatialProcessingStrategy:
|
||||
self.data_processor.figs["windrose"] = plotting.windrose(self.data_processor.df, plot_transect=True)
|
||||
else:
|
||||
self.data_processor.figs["windrose"] = plotting.windrose(self.data_processor.df)
|
||||
self.data_processor.figs["wind_timeseries"] = plotting.time_series(
|
||||
self.data_processor.df, ys=["windspeed", "winddir"]
|
||||
)
|
||||
self.data_processor.figs["scatter_3d"][gas] = None
|
||||
self.data_processor.figs["windrose"] = None
|
||||
self.data_processor.figs["wind_timeseries"] = None
|
||||
|
||||
|
||||
class SpatialProcessingStrategy(ABC):
|
||||
@ -164,15 +156,17 @@ class CurtainSpatialProcessingStrategy(SpatialProcessingStrategy):
|
||||
for gas_name in self.data_processor.gases:
|
||||
#计算通量
|
||||
self.data_processor.df = gas.gas_flux_column(self.data_processor.df, gas_name)
|
||||
self.data_processor.figs["scatter_3d"][gas_name].add_trace(
|
||||
go.Scatter3d(
|
||||
x=self.data_processor.dfs["removed"]["utm_easting"],
|
||||
y=self.data_processor.dfs["removed"]["utm_northing"],
|
||||
z=self.data_processor.dfs["removed"]["height_ato"],
|
||||
mode="markers",
|
||||
marker={"size": 2, "color": "black", "symbol": "circle", "opacity": 0.5},
|
||||
fig = self.data_processor.figs["scatter_3d"].get(gas_name)
|
||||
if fig is not None:
|
||||
fig.add_trace(
|
||||
go.Scatter3d(
|
||||
x=self.data_processor.dfs["removed"]["utm_easting"],
|
||||
y=self.data_processor.dfs["removed"]["utm_northing"],
|
||||
z=self.data_processor.dfs["removed"]["height_ato"],
|
||||
mode="markers",
|
||||
marker={"size": 2, "color": "black", "symbol": "circle", "opacity": 0.5},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class SpiralSpatialProcessingStrategy(SpatialProcessingStrategy):
|
||||
@ -203,15 +197,17 @@ class SpiralSpatialProcessingStrategy(SpatialProcessingStrategy):
|
||||
self.data_processor.df["x"] = self.data_processor.df["circumference_distance"]
|
||||
for gas_name in self.data_processor.gases:
|
||||
self.data_processor.df = gas.gas_flux_column(self.data_processor.df, gas_name)
|
||||
self.data_processor.figs["scatter_3d"][gas_name].add_trace(
|
||||
go.Scatter3d(
|
||||
x=self.data_processor.dfs["removed"]["utm_easting"],
|
||||
y=self.data_processor.dfs["removed"]["utm_northing"],
|
||||
z=self.data_processor.dfs["removed"]["height_ato"],
|
||||
mode="markers",
|
||||
marker={"size": 2, "color": "black", "symbol": "circle", "opacity": 0.5},
|
||||
fig = self.data_processor.figs["scatter_3d"].get(gas_name)
|
||||
if fig is not None:
|
||||
fig.add_trace(
|
||||
go.Scatter3d(
|
||||
x=self.data_processor.dfs["removed"]["utm_easting"],
|
||||
y=self.data_processor.dfs["removed"]["utm_northing"],
|
||||
z=self.data_processor.dfs["removed"]["height_ato"],
|
||||
mode="markers",
|
||||
marker={"size": 2, "color": "black", "symbol": "circle", "opacity": 0.5},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class InterpolationStrategy(ABC):
|
||||
|
||||
@ -2,15 +2,12 @@
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import plotly.graph_objects as go
|
||||
from jinja2 import Template
|
||||
from plotly.io import to_html
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import yaml
|
||||
|
||||
import logging
|
||||
from . import plotting
|
||||
|
||||
|
||||
import json
|
||||
@ -21,26 +18,23 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def mass_balance_report(
|
||||
krig_params: dict,
|
||||
wind_fig: go.Figure,
|
||||
background_fig: go.Figure,
|
||||
threed_fig: go.Figure,
|
||||
krig_fig: go.Figure,
|
||||
windrose_fig: go.Figure,
|
||||
wind_fig,
|
||||
background_fig,
|
||||
threed_fig,
|
||||
krig_fig,
|
||||
windrose_fig,
|
||||
) -> str:
|
||||
"""Generate a mass balance report."""
|
||||
"""Generate a mass balance report (plots disabled)."""
|
||||
template_path = Path(__file__).parent / "templates" / "mass_balance_template.html"
|
||||
|
||||
# Convert the figures to HTML
|
||||
plot_htmls = {}
|
||||
for name, fig in zip(
|
||||
["3D", "krig", "windrose", "wind", "background"],
|
||||
[threed_fig, krig_fig, windrose_fig, wind_fig, background_fig],
|
||||
strict=False,
|
||||
):
|
||||
if fig:
|
||||
plot_htmls[name] = to_html(fig, full_html=False)
|
||||
else:
|
||||
plot_htmls[name] = plotting.blank_figure()
|
||||
# Plots disabled -> use empty strings
|
||||
plot_htmls = {
|
||||
"3D": "",
|
||||
"krig": "",
|
||||
"windrose": "",
|
||||
"wind": "",
|
||||
"background": "",
|
||||
}
|
||||
|
||||
summary_data = {
|
||||
"Estimated flux": f"{krig_params.get('volume', 0):.3f} kgh⁻¹",
|
||||
|
||||
Reference in New Issue
Block a user