Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c6b9783a7a | |||
| 55cb7fb025 | |||
| adaabb2831 | |||
| 9bafc9ba0e | |||
| 9898b68410 | |||
| fa0cd769a8 | |||
| 3367774574 | |||
| 616783fb52 | |||
| 4ec46208dd | |||
| 5609a8d708 | |||
| e5f43f0297 | |||
| 408e1660fc | |||
| e3acd46da9 | |||
| d755367df0 | |||
| e16bd2976f | |||
| b7e389c3d7 | |||
| 72a20adc87 | |||
| 202eb238c7 |
20
.idea/claudeCodeTabState.xml
generated
Normal file
20
.idea/claudeCodeTabState.xml
generated
Normal file
@ -0,0 +1,20 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ClaudeCodeTabState">
|
||||
<option name="tabSessions">
|
||||
<map>
|
||||
<entry key="0">
|
||||
<value>
|
||||
<TabSessionState>
|
||||
<option name="provider" value="claude" />
|
||||
<option name="sessionId" value="cf5393e7-6de9-4f60-bde3-38759f3d4dba" />
|
||||
<option name="cwd" value="$PROJECT_DIR$" />
|
||||
<option name="model" value="claude-opus-4-8[1m]" />
|
||||
<option name="permissionMode" value="bypassPermissions" />
|
||||
</TabSessionState>
|
||||
</value>
|
||||
</entry>
|
||||
</map>
|
||||
</option>
|
||||
</component>
|
||||
</project>
|
||||
17
.idea/dataSources.xml
generated
Normal file
17
.idea/dataSources.xml
generated
Normal file
@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="gasflux" uuid="bc1d152a-2bb7-4a37-b6db-376527f2decb">
|
||||
<driver-ref>sqlite.xerial</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
|
||||
<jdbc-url>jdbc:sqlite:D:\111\office\ZHLduijie\6\UAV-CO2\web_api_data\outputs\gasflux.db</jdbc-url>
|
||||
<working-dir>$ProjectFileDir$</working-dir>
|
||||
<libraries>
|
||||
<library>
|
||||
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.51.1/org/xerial/sqlite-jdbc/3.51.1.0/sqlite-jdbc-3.51.1.0.jar</url>
|
||||
</library>
|
||||
</libraries>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
||||
@ -5,19 +5,19 @@ REM This script builds a standalone executable using Waitress WSGI server
|
||||
echo Building GasFlux Web API executable...
|
||||
|
||||
REM Check if PyInstaller is installed
|
||||
python -c "import PyInstaller" >nul 2>&1
|
||||
py -3 -c "import PyInstaller" >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo Error: PyInstaller is not installed. Please run:
|
||||
echo pip install pyinstaller waitress
|
||||
echo py -3 -m pip install pyinstaller waitress
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
REM Check if Waitress is installed
|
||||
python -c "import waitress" >nul 2>&1
|
||||
py -3 -c "import waitress" >nul 2>&1
|
||||
if errorlevel 1 (
|
||||
echo Error: Waitress is not installed. Please run:
|
||||
echo pip install waitress
|
||||
echo py -3 -m pip install waitress
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
@ -41,8 +41,24 @@ pyinstaller --onefile ^
|
||||
--hidden-import skgstat ^
|
||||
--hidden-import skgstat.Variogram ^
|
||||
--hidden-import skgstat.OrdinaryKriging ^
|
||||
--add-data "src\gasflux\gasflux_config.yaml;src\gasflux" ^
|
||||
--add-data "API_DOCUMENTATION.md;." ^
|
||||
--hidden-import skgstat.DirectionalVariogram ^
|
||||
--hidden-import plotly ^
|
||||
--hidden-import plotly.graph_objects ^
|
||||
--hidden-import plotly.express ^
|
||||
--hidden-import plotly.subplots ^
|
||||
--hidden-import pybaselines ^
|
||||
--hidden-import scipy ^
|
||||
--hidden-import scipy.odr ^
|
||||
--hidden-import scipy.signal ^
|
||||
--hidden-import scipy.stats ^
|
||||
--hidden-import scipy.integrate ^
|
||||
--hidden-import scipy.optimize ^
|
||||
--hidden-import molmass ^
|
||||
--hidden-import openpyxl ^
|
||||
--hidden-import pyproj ^
|
||||
--hidden-import jinja2 ^
|
||||
--hidden-import requests ^
|
||||
--hidden-import urllib3 ^
|
||||
--hidden-import matplotlib ^
|
||||
--hidden-import matplotlib.pyplot ^
|
||||
--hidden-import matplotlib.backends ^
|
||||
@ -53,7 +69,19 @@ pyinstaller --onefile ^
|
||||
--hidden-import matplotlib.patches ^
|
||||
--hidden-import matplotlib.text ^
|
||||
--hidden-import matplotlib.transforms ^
|
||||
--hidden-import tqdm ^
|
||||
--hidden-import certifi ^
|
||||
--hidden-import charset_normalizer ^
|
||||
--hidden-import geopandas ^
|
||||
--hidden-import shapely ^
|
||||
--hidden-import fiona ^
|
||||
--hidden-import simplekml ^
|
||||
--hidden-import joblib ^
|
||||
--exclude-module tkinter ^
|
||||
--add-data "src\gasflux\gasflux_config.yaml;src\gasflux" ^
|
||||
--add-data "src\gasflux\templates\mass_balance_template.html;src\gasflux\templates" ^
|
||||
--add-data "gasflux.ini;." ^
|
||||
--add-data "gasflux.ini.example;." ^
|
||||
server_waitress.py
|
||||
|
||||
if errorlevel 1 (
|
||||
@ -67,8 +95,10 @@ echo Build completed successfully!
|
||||
echo Executable created: dist\GasFluxAPI.exe
|
||||
echo.
|
||||
echo To run the server:
|
||||
echo GasFluxAPI.exe
|
||||
echo GasFluxAPI.exe
|
||||
echo.
|
||||
echo The server will start on http://localhost:5000
|
||||
echo The server will start on http://localhost:5001
|
||||
echo Make sure gasflux.ini is in the same directory as the executable,
|
||||
echo or configure paths via environment variables.
|
||||
echo.
|
||||
pause
|
||||
pause
|
||||
|
||||
@ -30,9 +30,9 @@ task_cleanup_interval = 30
|
||||
# 24 hours in seconds
|
||||
max_task_age = 86400
|
||||
# 1 minute in seconds for successful tasks
|
||||
successful_task_cleanup_age = 60
|
||||
successful_task_cleanup_age = 315360000
|
||||
# 1 minute in seconds for failed tasks
|
||||
failed_task_cleanup_age = 60
|
||||
failed_task_cleanup_age = 315360000
|
||||
janitor_dry_run = false
|
||||
|
||||
[performance]
|
||||
|
||||
107
requirements.txt
107
requirements.txt
@ -1,21 +1,112 @@
|
||||
# Automatically generated by https://github.com/damnever/pigar.
|
||||
|
||||
aiofiles==25.1.0
|
||||
altgraph==0.17.5
|
||||
amqp==5.3.1
|
||||
aniso8601==10.0.1
|
||||
annotated-doc==0.0.4
|
||||
annotated-types==0.7.0
|
||||
anyio==4.12.1
|
||||
async-timeout==5.0.1
|
||||
attrs==25.4.0
|
||||
backports.asyncio.runner==1.2.0
|
||||
billiard==4.2.4
|
||||
blinker==1.9.0
|
||||
celery==5.6.2
|
||||
certifi==2026.1.4
|
||||
charset-normalizer==3.4.4
|
||||
click==8.3.1
|
||||
click-didyoumean==0.3.1
|
||||
click-plugins==1.1.1.2
|
||||
click-repl==0.3.0
|
||||
cligj==0.7.2
|
||||
colorama==0.4.6
|
||||
contourpy==1.3.2
|
||||
coverage==7.13.1
|
||||
cycler==0.12.1
|
||||
et_xmlfile==2.0.0
|
||||
exceptiongroup==1.3.1
|
||||
fastapi==0.128.0
|
||||
fiona==1.10.1
|
||||
Flask==3.1.2
|
||||
flask-cors==6.0.2
|
||||
flask-restx==1.3.2
|
||||
fonttools==4.61.1
|
||||
geographiclib==2.1
|
||||
geopandas==0.14.3
|
||||
geopy==2.4.1
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httpx==0.28.1
|
||||
idna==3.11
|
||||
ImageIO==2.37.2
|
||||
importlib_resources==6.5.2
|
||||
iniconfig==2.3.0
|
||||
itsdangerous==2.2.0
|
||||
Jinja2==3.1.6
|
||||
joblib==1.3.2
|
||||
jsonschema==4.26.0
|
||||
jsonschema-specifications==2025.9.1
|
||||
kiwisolver==1.4.9
|
||||
kombu==5.6.2
|
||||
lazy_loader==0.4
|
||||
llvmlite==0.46.0
|
||||
MarkupSafe==3.0.3
|
||||
matplotlib==3.10.0
|
||||
molmass==2023.8.30
|
||||
narwhals==2.14.0
|
||||
networkx==3.4.2
|
||||
numba==0.63.1
|
||||
numpy==2.1.3
|
||||
openpyxl==3.1.5
|
||||
packaging==25.0
|
||||
pandas==2.2.3
|
||||
pefile==2024.8.26
|
||||
pillow==12.1.0
|
||||
plotly==5.20.0
|
||||
pluggy==1.6.0
|
||||
prompt_toolkit==3.0.52
|
||||
psutil==7.2.1
|
||||
pybaselines==1.1.0
|
||||
pyproj==3.6.1
|
||||
pydantic==2.12.5
|
||||
pydantic_core==2.41.5
|
||||
Pygments==2.19.2
|
||||
pyinstaller==6.17.0
|
||||
pyinstaller-hooks-contrib==2025.11
|
||||
pyogrio==0.12.1
|
||||
pyparsing==3.3.1
|
||||
pyproj==3.7.1
|
||||
pytest==8.1.1
|
||||
pytest-asyncio
|
||||
pytest-cov==7.0.0
|
||||
pytest-mock==3.15.1
|
||||
python-dateutil==2.9.0.post0
|
||||
python-multipart==0.0.21
|
||||
pytz==2025.2
|
||||
pywin32-ctypes==0.2.3
|
||||
PyYAML==6.0.1
|
||||
redis==7.1.0
|
||||
referencing==0.37.0
|
||||
requests==2.32.5
|
||||
rpds-py==0.30.0
|
||||
scikit-gstat==1.0.19
|
||||
scikit-image==0.24.0
|
||||
scikit-learn==1.7.2
|
||||
scipy==1.15.1
|
||||
flask
|
||||
werkzeug
|
||||
flask-cors
|
||||
waitress
|
||||
psutil
|
||||
shapely==2.1.2
|
||||
simplekml==1.3.6
|
||||
six==1.17.0
|
||||
starlette==0.50.0
|
||||
tenacity==9.1.2
|
||||
threadpoolctl==3.6.0
|
||||
tifffile==2025.5.10
|
||||
tomli==2.4.0
|
||||
tqdm==4.67.1
|
||||
typing-inspection==0.4.2
|
||||
typing_extensions==4.15.0
|
||||
tzdata==2025.3
|
||||
tzlocal==5.3.1
|
||||
urllib3==2.6.2
|
||||
uvicorn==0.40.0
|
||||
vine==5.1.0
|
||||
waitress==3.0.2
|
||||
wcwidth==0.2.14
|
||||
Werkzeug==3.1.5
|
||||
|
||||
@ -45,9 +45,17 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 设置控制台编码为 UTF-8(解决 Windows 中文乱码问题)
|
||||
# PyInstaller onefile 模式下 sys.stdout.buffer 可能不存在(无控制台或管道重定向),
|
||||
# 此时跳过编码包装,避免 AttributeError 导致启动崩溃
|
||||
import io
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
try:
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
except (AttributeError, ValueError):
|
||||
pass
|
||||
try:
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
except (AttributeError, ValueError):
|
||||
pass
|
||||
|
||||
# Add the project root and src directory to PYTHONPATH
|
||||
project_root = Path(__file__).parent.absolute()
|
||||
|
||||
@ -397,20 +397,11 @@ def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
logger.error(f"Job {task_id}: Failed to load config from {config_path}: {str(e)}")
|
||||
raise
|
||||
|
||||
# 根据配置 output_dir 更新目录
|
||||
Config.update_directories_from_config(config_path)
|
||||
# 不再全局覆盖 Config 类属性(避免多用户并发冲突)
|
||||
# 每个任务使用独立目录,基础路径由 INI 配置决定,task_id 保证隔离
|
||||
# Config.update_directories_from_config(config_path) ← 已移除
|
||||
|
||||
# 将更新后的目录同步到 app.config
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
|
||||
# 任务状态持久化现在仅使用 SQLite
|
||||
# JSON 持久化已禁用
|
||||
# from .shared import set_task_status_file_path, load_task_status_from_file
|
||||
# set_task_status_file_path(Config.OUTPUT_FOLDER / "task_status.json")
|
||||
# load_task_status_from_file()
|
||||
|
||||
# 更新任务目录到正确的基于配置的路径下
|
||||
# 使用 INI 配置的基础目录 + task_id 构建任务专属路径
|
||||
from pathlib import Path
|
||||
job_upload_dir = Path(Config.UPLOAD_FOLDER) / task_id
|
||||
job_output_dir = Path(Config.OUTPUT_FOLDER) / task_id
|
||||
|
||||
@ -52,7 +52,11 @@ def algorithmic_baseline(
|
||||
signal = (df[gas] - bkg)[~bkg_points]
|
||||
df[f"{gas}_signal"] = np.invert(bkg_points)
|
||||
|
||||
fig = None # plotting disabled
|
||||
try:
|
||||
from . import plotting
|
||||
fig = plotting.background_plotting(df, gas)
|
||||
except Exception:
|
||||
fig = None
|
||||
output_text = (
|
||||
f"Baseline algorithm: {algorithm}\n"
|
||||
f"Positive and negative 95% percentile of baseline: {np.percentile(background, 2.5):.2f} ppm, "
|
||||
|
||||
@ -3,6 +3,7 @@ Download Blueprint
|
||||
Handles file download endpoints.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from flask import Blueprint, send_file, current_app
|
||||
|
||||
@ -10,40 +11,6 @@ from ..shared import _format_response, log_performance, logger
|
||||
from ..auth import require_api_key
|
||||
|
||||
|
||||
def _mark_task_downloaded(task_id):
|
||||
"""Mark task as downloaded and schedule deletion in database."""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
# Use independent database connection (not from flask.g which may be closed)
|
||||
from ..db import get_db_path as get_config_db_path
|
||||
db_path = get_config_db_path(current_app)
|
||||
|
||||
# Get cleanup age for successful tasks from config (in seconds)
|
||||
successful_task_cleanup_age = current_app.config.get('SUCCESSFUL_TASK_CLEANUP_AGE', 3600)
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path), check_same_thread=False)
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=3000")
|
||||
|
||||
# Update downloaded timestamp and set deletion time based on config
|
||||
conn.execute("""
|
||||
UPDATE tasks
|
||||
SET downloaded_at = datetime('now', '+8 hours'),
|
||||
delete_after_at = datetime('now', '+8 hours', '+' || ? || ' seconds')
|
||||
WHERE task_id = ?
|
||||
""", (successful_task_cleanup_age, task_id))
|
||||
|
||||
conn.commit()
|
||||
logger.info(f"Task {task_id} marked as downloaded, scheduled for deletion in {successful_task_cleanup_age} seconds")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark task {task_id} as downloaded: {str(e)}", exc_info=True)
|
||||
finally:
|
||||
if 'conn' in locals():
|
||||
conn.close()
|
||||
|
||||
# Create blueprint
|
||||
download_bp = Blueprint('download', __name__, url_prefix='/download')
|
||||
|
||||
@ -124,12 +91,19 @@ def download_file(filename):
|
||||
file_size = file_path.stat().st_size
|
||||
logger.info(f"Serving file: {filename} ({file_size} bytes)")
|
||||
|
||||
# Mark download immediately before sending file
|
||||
# 记录下载时间(仅时间戳,不设置自动删除)
|
||||
if task_id:
|
||||
try:
|
||||
_mark_task_downloaded(task_id)
|
||||
from ..db import get_db
|
||||
db = get_db()
|
||||
db.execute(
|
||||
"UPDATE tasks SET downloaded_at = datetime('now', '+8 hours') WHERE task_id = ?",
|
||||
(task_id,)
|
||||
)
|
||||
db.commit()
|
||||
logger.info(f"Task {task_id} downloaded at {datetime.now()}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark download for task {task_id}: {str(e)}")
|
||||
logger.error(f"Failed to record download for task {task_id}: {str(e)}")
|
||||
|
||||
response = send_file(file_path)
|
||||
return response
|
||||
|
||||
@ -544,17 +544,17 @@ def calculate_pressure(df, max_samples=None, height_tolerance=10.0, height_bin_s
|
||||
df['pressure'] = None # 初始化
|
||||
df.loc[sample_df.index, 'pressure'] = pressures
|
||||
|
||||
# 对于未计算的行,使用插值或平均值填充
|
||||
if max_samples is not None and len(df) > max_samples:
|
||||
# 只计算了部分行,用平均值填充其余行
|
||||
valid_pressures_for_mean = [p for p in pressures if p is not None]
|
||||
if valid_pressures_for_mean:
|
||||
mean_pressure = sum(valid_pressures_for_mean) / len(valid_pressures_for_mean)
|
||||
df['pressure'] = df['pressure'].fillna(mean_pressure)
|
||||
print(f"使用平均气压填充其余 {len(df) - max_samples} 行: {mean_pressure:.1f} hPa")
|
||||
# 统计并填充缺失气压值:用已有有效值的平均值填充所有NaN
|
||||
valid_pressures = [p for p in pressures if p is not None]
|
||||
nan_count_before = df['pressure'].isna().sum()
|
||||
|
||||
if valid_pressures:
|
||||
mean_pressure = sum(valid_pressures) / len(valid_pressures)
|
||||
df['pressure'] = df['pressure'].fillna(mean_pressure)
|
||||
if nan_count_before > 0:
|
||||
print(f"使用平均气压 {mean_pressure:.1f} hPa 填充了 {nan_count_before} 行缺失气压值")
|
||||
|
||||
# 统计信息
|
||||
valid_pressures = [p for p in pressures if p is not None]
|
||||
if valid_pressures:
|
||||
avg_pressure = sum(valid_pressures) / len(valid_pressures)
|
||||
print(f"成功计算 {len(valid_pressures)}/{actual_samples} 个气压值,平均值: {avg_pressure:.1f} hPa")
|
||||
|
||||
@ -33,7 +33,7 @@ def gas_density(local_pressure: float, local_temperature_celsius: float, gas: st
|
||||
local_volume = (
|
||||
gas_variables["standard_molar_volume"]
|
||||
* (gas_variables["standard_pressure"] / local_pressure)
|
||||
* ((local_temperature_kelvin + gas_variables["standard_temperature"]) / gas_variables["standard_temperature"])
|
||||
* (local_temperature_kelvin / gas_variables["standard_temperature"])
|
||||
) # m3⋅mol-1
|
||||
return mass(gas) / 1000 / local_volume
|
||||
|
||||
|
||||
@ -99,11 +99,6 @@ def ordinary_kriging(
|
||||
# np.nan_to_num(error_1s, copy=False, nan=0)
|
||||
volume_error = simpsonintegrate(error_1s, x_cell_size, y_cell_size)
|
||||
|
||||
# Plots disabled
|
||||
contour_plot = None
|
||||
grid_plot = None
|
||||
semivariogram_plot = None
|
||||
|
||||
output_text = (
|
||||
f"The emissions flux of {gas.upper()} is {volume:.3f}kgh⁻¹; "
|
||||
f"the cut and fill volumes of the grid are {volumepos:.3f} and {volumeneg:.3f}kgh⁻¹. "
|
||||
@ -123,6 +118,20 @@ def ordinary_kriging(
|
||||
"volume_error": volume_error,
|
||||
}
|
||||
|
||||
# Generate plots (must be after krig_variables is defined)
|
||||
try:
|
||||
contour_plot = plotting._contour_krig_wrapper(krig_variables)
|
||||
except Exception:
|
||||
contour_plot = None
|
||||
try:
|
||||
grid_plot = plotting._heatmap_krig_wrapper(krig_variables)
|
||||
except Exception:
|
||||
grid_plot = None
|
||||
try:
|
||||
semivariogram_plot = plotting._semivariogram_plot(semivariogram, gas)
|
||||
except Exception:
|
||||
semivariogram_plot = None
|
||||
|
||||
return krig_variables, output_text, contour_plot, grid_plot, semivariogram_plot
|
||||
|
||||
|
||||
|
||||
@ -79,7 +79,7 @@ def cleanup_expired_tasks():
|
||||
""").fetchall()
|
||||
|
||||
if rows:
|
||||
current_app.logger.info(f"Janitor found {len(rows)} expired tasks to clean up")
|
||||
current_app.logger.info(f"Janitor found {len(rows)} expired tasks to clean up")
|
||||
|
||||
for row in rows:
|
||||
task_id = row['task_id']
|
||||
@ -310,8 +310,7 @@ def reconcile_tasks_on_startup():
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Failed to delete failed task {task_id}: {str(e)}", exc_info=True)
|
||||
|
||||
# Check for tasks with output directories that no longer exist
|
||||
# This helps clean up database entries for manually deleted directories
|
||||
# 检查 output_dir 不存在的任务(仅警告,不自动删除)
|
||||
rows = conn.execute("""
|
||||
SELECT task_id, output_dir
|
||||
FROM tasks
|
||||
@ -321,18 +320,21 @@ def reconcile_tasks_on_startup():
|
||||
|
||||
orphaned_count = 0
|
||||
for row in rows:
|
||||
task_id_from_db = row[0] # task_id是第一个字段
|
||||
output_dir_from_db = row[1] # output_dir是第二个字段
|
||||
task_id_from_db = row[0]
|
||||
output_dir_from_db = row[1]
|
||||
|
||||
if not Path(output_dir_from_db).exists():
|
||||
conn.execute(
|
||||
"DELETE FROM tasks WHERE task_id = ?",
|
||||
(task_id_from_db,)
|
||||
current_app.logger.warning(
|
||||
f"Startup reconciliation: Task {task_id_from_db} output directory "
|
||||
f"not found on disk: {output_dir_from_db}. Database record preserved."
|
||||
)
|
||||
orphaned_count += 1
|
||||
|
||||
if orphaned_count > 0:
|
||||
current_app.logger.info(f"Startup reconciliation: Marked {orphaned_count} tasks as deleted (directories not found)")
|
||||
current_app.logger.info(
|
||||
f"Startup reconciliation: Found {orphaned_count} tasks with missing directories "
|
||||
f"(records preserved, use DELETE /task/<id> to manually clean up)"
|
||||
)
|
||||
|
||||
# Backfill output_dir for rows with NULL, using OUTPUT_FOLDER/<task_id>
|
||||
output_base = Path(current_app.config.get('OUTPUT_FOLDER') or '')
|
||||
|
||||
@ -1,44 +1,718 @@
|
||||
"""
|
||||
Lightweight stub plotting module to disable heavy visualization dependencies.
|
||||
All functions return None so callers can safely check for truthiness.
|
||||
"""
|
||||
"""Various plotting functions mainly based around plotly."""
|
||||
|
||||
import matplotlib.colors as mcolors
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import plotly.io as pio
|
||||
import simplekml
|
||||
from plotly.subplots import make_subplots
|
||||
|
||||
from . import processing
|
||||
|
||||
pio.templates["default"] = go.layout.Template(
|
||||
layout=go.Layout(
|
||||
margin=go.layout.Margin(l=0, r=0, b=0, t=0, pad=0),
|
||||
),
|
||||
)
|
||||
|
||||
pio.templates.default = "simple_white+default"
|
||||
|
||||
|
||||
styling = {
|
||||
"colorscale": "geyser",
|
||||
}
|
||||
|
||||
|
||||
def blank_figure():
|
||||
return None
|
||||
fig = go.Figure()
|
||||
return fig
|
||||
|
||||
|
||||
def scatter_3d(*args, **kwargs):
|
||||
return None
|
||||
def scatter_3d(
|
||||
df: pd.DataFrame,
|
||||
color: str = "",
|
||||
colorbar_title: str = "",
|
||||
timestamp: str = "timestamp",
|
||||
x: str = "utm_easting",
|
||||
y: str = "utm_northing",
|
||||
z: str = "height_ato",
|
||||
courses: bool = False,
|
||||
):
|
||||
fig = px.scatter_3d(df, x=x, y=y, z=z)
|
||||
|
||||
if color:
|
||||
custom_data = [df[timestamp]]
|
||||
if courses:
|
||||
custom_data.extend([df["course_elevation"], df["course_azimuth"]])
|
||||
custom_data = np.stack(custom_data, axis=-1)
|
||||
hover_template = [
|
||||
f"{x}: %{{x:.2f}}",
|
||||
f"{y}: %{{y:.2f}}",
|
||||
f"{z}: %{{z:.2f}}",
|
||||
f"{color}: %{{marker.color:.2f}}",
|
||||
f"{timestamp}: %{{customdata[0]|%Y-%m-%d %H:%M:%S}}",
|
||||
"Index: %{pointNumber}",
|
||||
]
|
||||
|
||||
if courses:
|
||||
hover_template.extend(
|
||||
[
|
||||
"Course Elevation: %{customdata[1]:.2f}",
|
||||
"Course Azimuth: %{customdata[2]:.2f}",
|
||||
]
|
||||
)
|
||||
hover_template_str = "<br>".join(hover_template)
|
||||
fig.update_traces(
|
||||
marker=dict(
|
||||
color=df[color],
|
||||
size=4,
|
||||
opacity=0.5,
|
||||
colorscale=styling["colorscale"],
|
||||
colorbar=dict(title=colorbar_title),
|
||||
),
|
||||
customdata=custom_data,
|
||||
hovertemplate=hover_template_str,
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def scatter_2d(*args, **kwargs):
|
||||
return None
|
||||
def scatter_2d(
|
||||
df: pd.DataFrame,
|
||||
x: str,
|
||||
color: str,
|
||||
y: str = "height_ato",
|
||||
**kwargs,
|
||||
):
|
||||
fig = px.scatter(
|
||||
df,
|
||||
x=x,
|
||||
y=y,
|
||||
color=color,
|
||||
color_continuous_scale=styling["colorscale"],
|
||||
opacity=0.8,
|
||||
**kwargs,
|
||||
)
|
||||
fig.update_traces(
|
||||
customdata=df.index,
|
||||
hovertemplate="<br>".join(
|
||||
[
|
||||
"x: %{x:.2f}",
|
||||
"height_ato: %{y:.2f}",
|
||||
f"{color}: %{{marker.color:.2f}}",
|
||||
"Time: %{customdata}",
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def time_series(*args, **kwargs):
|
||||
return None
|
||||
def time_series(
|
||||
df: pd.DataFrame,
|
||||
ys: str | list[str],
|
||||
x: str = "timestamp",
|
||||
color: str | None = None,
|
||||
split=None,
|
||||
y_mins: float | list[float | int] | None = None,
|
||||
rolling_average: bool = True,
|
||||
scatter: bool = True,
|
||||
rolling_window: int = 5,
|
||||
y_titles: str | list[str] | None = None,
|
||||
legend: bool = True,
|
||||
) -> go.Figure:
|
||||
colors = px.colors.qualitative.Plotly
|
||||
|
||||
if isinstance(ys, str):
|
||||
ys = [ys]
|
||||
if y_titles is None:
|
||||
y_titles = ys
|
||||
single_title = False
|
||||
elif isinstance(y_titles, str):
|
||||
y_titles = [y_titles]
|
||||
single_title = True
|
||||
elif isinstance(y_titles, list):
|
||||
if len(y_titles) != len(ys):
|
||||
raise ValueError("Length of y_titles must be equal to length of ys")
|
||||
single_title = False
|
||||
else:
|
||||
raise ValueError("Invalid y_titles value")
|
||||
if isinstance(y_mins, (float | int)):
|
||||
y_mins = [y_mins]
|
||||
if isinstance(y_mins, list):
|
||||
if len(y_mins) != len(ys):
|
||||
raise ValueError("Length of y_mins must be equal to length of ys")
|
||||
|
||||
fig = go.Figure()
|
||||
|
||||
axis_space = 0.05
|
||||
domain_start = axis_space * (len(ys)) if len(ys) > 1 else 0
|
||||
fig.update_layout(
|
||||
xaxis=dict(
|
||||
domain=[domain_start, 1],
|
||||
),
|
||||
)
|
||||
|
||||
for i, y in enumerate(ys):
|
||||
yaxis_name = f"yaxis{i+1}"
|
||||
yaxis_ref = f"y{i+1}"
|
||||
|
||||
trace_color = "black" if single_title and i == 0 else colors[i % len(colors)]
|
||||
|
||||
marker_i = dict(size=8, opacity=0.3 if rolling_average else 0.5, color=trace_color)
|
||||
if color is not None:
|
||||
marker_i["color"] = df[color] # type: ignore
|
||||
marker_i["colorscale"] = styling["colorscale"]
|
||||
|
||||
hover_template = f"{x}: %{{x}}<br>{y}: %{{y:.2f}}<br>"
|
||||
if color:
|
||||
hover_template += f"{color}: %{{marker.color:.2f}}<br>"
|
||||
|
||||
if scatter:
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[y],
|
||||
name=y,
|
||||
mode="markers",
|
||||
marker=marker_i,
|
||||
yaxis=yaxis_ref,
|
||||
hovertemplate=hover_template,
|
||||
showlegend=legend,
|
||||
)
|
||||
)
|
||||
|
||||
if rolling_average:
|
||||
df[f"rolling_avg_{i}"] = df[y].rolling(window=rolling_window, min_periods=1).mean()
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[f"rolling_avg_{i}"],
|
||||
name=f"{y} {rolling_window}-point avg",
|
||||
mode="lines",
|
||||
line=dict(color=trace_color, width=2),
|
||||
yaxis=yaxis_ref,
|
||||
showlegend=legend,
|
||||
)
|
||||
)
|
||||
|
||||
y_data = df[y]
|
||||
y_min_var = y_data.min()
|
||||
y_max_var = y_data.max()
|
||||
y_range = y_max_var - y_min_var or y_max_var * 0.05
|
||||
|
||||
y_axis_min = y_mins[i] if y_mins is not None and y_mins[i] is not None else y_min_var - y_range * 0.05
|
||||
y_axis_max = y_max_var + y_range * 0.05
|
||||
|
||||
if single_title and i == 0:
|
||||
axis_title = dict(text=y_titles[0], font=dict(color="black"))
|
||||
elif not single_title:
|
||||
axis_title = dict(text=y_titles[i], font=dict(color=trace_color))
|
||||
else:
|
||||
axis_title = None
|
||||
|
||||
axis_config = dict(
|
||||
title=axis_title,
|
||||
tickfont=dict(color=trace_color),
|
||||
range=[y_axis_min, y_axis_max],
|
||||
side="left",
|
||||
position=axis_space * i if i > 0 else None,
|
||||
anchor="free" if i > 0 else None,
|
||||
overlaying="y" if i > 0 else None,
|
||||
showgrid=(i == 0),
|
||||
)
|
||||
|
||||
fig.layout[yaxis_name] = axis_config
|
||||
|
||||
if split is not None:
|
||||
fig.add_shape(
|
||||
type="line",
|
||||
xref="x",
|
||||
yref="paper",
|
||||
x0=split,
|
||||
y0=0,
|
||||
x1=split,
|
||||
y1=1,
|
||||
line=dict(color="red", width=2),
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def background_plotting(*args, **kwargs):
|
||||
return None
|
||||
def background_plotting(df: pd.DataFrame, gas: str):
|
||||
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
||||
ymin = df[gas].min()
|
||||
ymax = df[gas].max()
|
||||
ylim = [ymin * 0.95, ymax * 1.05]
|
||||
y2min = df[f"{gas}_normalised"].min()
|
||||
y2lim = (y2min, y2min + (ylim[1] - ylim[0]))
|
||||
fig.update_yaxes(range=ylim, secondary_y=False, title_text=f"Sensor {gas} (ppm)")
|
||||
fig.update_yaxes(range=y2lim, secondary_y=True, title_text=f"Normalised {gas} (ppm)")
|
||||
fig.add_scatter(x=df["timestamp"], y=df[gas], opacity=0.3, name="Raw Data")
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"], y=df[f"{gas}_fit"], mode="lines", name="Fitted Background", line=dict(dash="dash")
|
||||
)
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"], y=df[f"{gas}_normalised"], yaxis="y2", name="Normalised Data", mode="lines", opacity=0.5
|
||||
)
|
||||
fig.add_scatter(
|
||||
x=df["timestamp"],
|
||||
y=np.where(df[f"{gas}_signal"], df[f"{gas}_normalised"], np.nan),
|
||||
yaxis="y2",
|
||||
name="Classed as signal",
|
||||
mode="lines",
|
||||
opacity=0.5,
|
||||
# color
|
||||
# mode="markers",
|
||||
# marker=dict(size=3),
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def windrose(*args, **kwargs):
|
||||
return None
|
||||
def windrose_process(df: pd.DataFrame):
|
||||
beaufort = {
|
||||
"0": [0, 1],
|
||||
"1": [1, 2],
|
||||
"2": [2, 4],
|
||||
"3": [4, 6],
|
||||
"4": [6, 9],
|
||||
"5": [9, 11],
|
||||
"6": [11, 14],
|
||||
"7": [14, 17],
|
||||
"8": [17, 21],
|
||||
"9": [21, 25],
|
||||
"10": [25, 29],
|
||||
"11": [29, 33],
|
||||
"12": [33, 200],
|
||||
}
|
||||
|
||||
beaufort_ms = {
|
||||
"0": "0-1",
|
||||
"1": "1-2",
|
||||
"2": "2-4",
|
||||
"3": "4-6",
|
||||
"4": "6-9",
|
||||
"5": "9-11",
|
||||
"6": "11-14",
|
||||
"7": "14-17",
|
||||
"8": "17-21",
|
||||
"9": "21-25",
|
||||
"10": "25-29",
|
||||
"11": "29-33",
|
||||
"12": "33+",
|
||||
}
|
||||
|
||||
cardinals = {
|
||||
"N1": [0, 11.25],
|
||||
"NNE": [11.25, 33.75],
|
||||
"NE": [33.75, 56.25],
|
||||
"ENE": [56.25, 78.75],
|
||||
"E": [78.75, 101.25],
|
||||
"ESE": [101.25, 123.75],
|
||||
"SE": [123.75, 146.25],
|
||||
"SSE": [146.25, 168.75],
|
||||
"S": [168.75, 191.25],
|
||||
"SSW": [191.25, 213.75],
|
||||
"SW": [213.75, 236.25],
|
||||
"WSW": [236.25, 258.75],
|
||||
"W": [258.75, 281.25],
|
||||
"WNW": [281.25, 303.75],
|
||||
"NW": [303.75, 326.25],
|
||||
"NNW": [326.25, 348.75],
|
||||
"N2": [348.75, 360],
|
||||
}
|
||||
df["wind_direction_bin"] = pd.cut(
|
||||
df["winddir"],
|
||||
bins=[lower for lower, upper in cardinals.values()] + [list(cardinals.values())[-1][1]],
|
||||
labels=[key for key in cardinals],
|
||||
right=False,
|
||||
)
|
||||
|
||||
df["wind_direction_bin"] = (
|
||||
df["wind_direction_bin"].map(lambda x: "N" if x in ["N1", "N2"] else x).astype("category")
|
||||
)
|
||||
df["beaufort"] = pd.cut(
|
||||
df["windspeed"],
|
||||
bins=[lower for lower, upper in beaufort.values()] + [list(beaufort.values())[-1][1]],
|
||||
labels=[key for key in beaufort],
|
||||
right=False,
|
||||
)
|
||||
df["beaufort_ms"] = df["beaufort"].map(beaufort_ms)
|
||||
df_windrose = df.groupby(["wind_direction_bin", "beaufort"], observed=False).size().reset_index(name="count") # type: ignore
|
||||
df_windrose["frequency"] = df_windrose["count"] / df_windrose["count"].sum() * 100
|
||||
df_windrose["wind_direction_bin_degs"] = df_windrose["wind_direction_bin"].cat.rename_categories(
|
||||
{
|
||||
"N": 0,
|
||||
"NNE": 22.5,
|
||||
"NE": 45,
|
||||
"ENE": 67.5,
|
||||
"E": 90,
|
||||
"ESE": 112.5,
|
||||
"SE": 135,
|
||||
"SSE": 157.5,
|
||||
"S": 180,
|
||||
"SSW": 202.5,
|
||||
"SW": 225,
|
||||
"WSW": 247.5,
|
||||
"W": 270,
|
||||
"WNW": 292.5,
|
||||
"NW": 315,
|
||||
"NNW": 337.5,
|
||||
},
|
||||
)
|
||||
df_windrose["beaufort"] = df_windrose["beaufort"].astype(int)
|
||||
return df_windrose
|
||||
|
||||
|
||||
def outliers(*args, **kwargs):
|
||||
return None
|
||||
def windrose_graph(df, plot_transect=False, theta1=None, theta2=None):
|
||||
n_colors = 13
|
||||
colors = px.colors.sample_colorscale("turbo", [n / (n_colors - 1) for n in range(n_colors)])
|
||||
fig = px.bar_polar(
|
||||
df,
|
||||
r="frequency",
|
||||
theta="wind_direction_bin_degs",
|
||||
color="beaufort",
|
||||
labels={
|
||||
"frequency": "Frequency (%)",
|
||||
"wind_direction_bin": "Direction",
|
||||
"beaufort": "Beaufort Scale",
|
||||
},
|
||||
color_discrete_map=colors,
|
||||
)
|
||||
fig.update_layout(polar=dict(radialaxis={"visible": False, "showticklabels": False}))
|
||||
fig.update_layout(
|
||||
polar=dict(
|
||||
angularaxis={
|
||||
"showgrid": False,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
fig.update_layout(polar_bargap=0)
|
||||
if plot_transect:
|
||||
max_freq = df.groupby("wind_direction_bin", observed=False)["frequency"].sum().max()
|
||||
fig.add_trace(
|
||||
go.Scatterpolar(
|
||||
r=[max_freq, max_freq],
|
||||
theta=[theta1, theta2],
|
||||
mode="lines",
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
showlegend=False,
|
||||
),
|
||||
)
|
||||
return fig
|
||||
|
||||
|
||||
def contour_krig(*args, **kwargs):
|
||||
return None
|
||||
def windrose(df: pd.DataFrame, plot_transect=False):
|
||||
df_windrose = windrose_process(df)
|
||||
if plot_transect:
|
||||
theta1, theta2 = processing.bimodal_azimuth(df)
|
||||
fig = windrose_graph(df_windrose, plot_transect=plot_transect, theta1=theta1, theta2=theta2)
|
||||
else:
|
||||
fig = windrose_graph(df_windrose, plot_transect=plot_transect)
|
||||
return fig
|
||||
|
||||
|
||||
def heatmap_krig(*args, **kwargs):
|
||||
return None
|
||||
def outliers(original_data: pd.Series, fence_high: float, fence_low: float):
|
||||
outliers = np.array(original_data > fence_high) | (original_data < fence_low)
|
||||
|
||||
fig = make_subplots(rows=1, cols=2, shared_yaxes=True)
|
||||
fig.add_trace(px.strip(original_data, color=outliers).data[0], row=1, col=1)
|
||||
if sum(outliers) > 0:
|
||||
fig.add_trace(px.strip(original_data, color=outliers).data[1], row=1, col=1)
|
||||
fig.add_shape(
|
||||
go.layout.Shape(
|
||||
type="line",
|
||||
x0=-0.5,
|
||||
y0=fence_high,
|
||||
x1=0.5,
|
||||
y1=fence_high,
|
||||
line=dict(color="red", width=2),
|
||||
),
|
||||
row=1,
|
||||
col=1,
|
||||
)
|
||||
fig.add_shape(
|
||||
go.layout.Shape(
|
||||
type="line",
|
||||
x0=-0.5,
|
||||
y0=fence_low,
|
||||
x1=0.5,
|
||||
y1=fence_low,
|
||||
line=dict(color="red", width=2),
|
||||
),
|
||||
row=1,
|
||||
col=1,
|
||||
)
|
||||
fig.update_traces(offsetgroup=0)
|
||||
|
||||
fig.add_trace(px.scatter(original_data, color=outliers).data[0], row=1, col=2)
|
||||
if sum(outliers) > 0:
|
||||
fig.add_trace(px.scatter(original_data, color=outliers).data[1], row=1, col=2)
|
||||
fig.update_layout(showlegend=False, yaxis_title="Windspeed (ms⁻¹)")
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def create_kml_file(*args, **kwargs):
|
||||
return None
|
||||
def contour_krig(
|
||||
df: pd.DataFrame,
|
||||
gas: str,
|
||||
# array of float 64
|
||||
xx: np.ndarray,
|
||||
yy: np.ndarray,
|
||||
field: np.ndarray,
|
||||
cut_ground: bool = False,
|
||||
x: str = "x",
|
||||
y: str = "height_ato",
|
||||
) -> go.Figure:
|
||||
if np.isnan(field).all():
|
||||
return blank_figure()
|
||||
fig = go.Figure()
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=df[x],
|
||||
y=df[y],
|
||||
mode="markers",
|
||||
marker={
|
||||
"color": df[f"{gas}_normalised"],
|
||||
"colorscale": styling["colorscale"],
|
||||
"showscale": True,
|
||||
"colorbar": {
|
||||
"title": f"{gas} (ppm)",
|
||||
},
|
||||
},
|
||||
showlegend=False,
|
||||
)
|
||||
)
|
||||
fig.add_trace(
|
||||
go.Contour(
|
||||
z=field.T,
|
||||
x=xx[:, 0],
|
||||
y=yy[0, :],
|
||||
contours={
|
||||
"start": field.min(),
|
||||
"end": field.max(),
|
||||
"size": (field[~np.isnan(field)].max() - field[~np.isnan(field)].min()) / 21,
|
||||
},
|
||||
colorscale=styling["colorscale"],
|
||||
opacity=0.5,
|
||||
showlegend=False,
|
||||
showscale=False,
|
||||
)
|
||||
)
|
||||
fig.update_xaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="horizontal distance on projected flux plane (m)",
|
||||
range=[np.min(xx), np.max(xx)],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=20,
|
||||
)
|
||||
fig.update_yaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="height above takeoff (m)",
|
||||
range=[np.min(yy), np.max(yy)],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=10,
|
||||
)
|
||||
if cut_ground:
|
||||
resolution = 200 # how many points to interpolate over
|
||||
df["ground_elevation_ato"] = df.loc[:, "height_ato"] - df.loc[:, "height_agl"]
|
||||
df_sorted = df.dropna(subset=[x, "ground_elevation_ato"]).sort_values(x)
|
||||
x_min, x_max = df_sorted[x].min(), df_sorted[x].max()
|
||||
x_interp = np.linspace(x_min, x_max, resolution)
|
||||
ground_ato_interp = np.interp(x_interp, df_sorted[x], df_sorted["ground_elevation_ato"])
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=x_interp,
|
||||
y=ground_ato_interp,
|
||||
mode="lines",
|
||||
line=dict(color="black", width=2, dash="dash"),
|
||||
name="Interpolated Ground Level",
|
||||
)
|
||||
)
|
||||
fig.layout.coloraxis.colorbar.title = "Emissions flux (kg⋅m⁻²⋅h⁻¹)"
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def heatmap_krig(xx: np.ndarray, yy: np.ndarray, field: np.ndarray):
|
||||
fig = px.imshow(field.T, x=xx[:, 0], y=yy[0, :], color_continuous_scale=styling["colorscale"], origin="lower")
|
||||
fig.layout.coloraxis.colorbar.title = "Emissions flux (kg⋅m⁻²⋅h⁻¹)"
|
||||
fig.update_xaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="horizontal distance on cylindrical projected flux plane (m)",
|
||||
range=[xx.min(), xx.max()],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=20,
|
||||
)
|
||||
fig.update_yaxes(
|
||||
showline=True,
|
||||
linewidth=1,
|
||||
linecolor="black",
|
||||
title_text="height above ground level (m)",
|
||||
range=[yy.min(), yy.max()],
|
||||
ticks="outside",
|
||||
tickwidth=1,
|
||||
tickcolor="black",
|
||||
ticklen=5,
|
||||
nticks=10,
|
||||
)
|
||||
fig.update_layout(coloraxis_colorbar=dict(len=0.25))
|
||||
return fig
|
||||
|
||||
|
||||
def create_kml_file(data: pd.DataFrame, output_file: str, column: str, altitudemode: str):
|
||||
kml = simplekml.Kml()
|
||||
|
||||
min_value = data[column].min()
|
||||
max_value = data[column].max()
|
||||
|
||||
custom_colors = [
|
||||
"#008080",
|
||||
"#70a494",
|
||||
"#b4c8a8",
|
||||
"#f6edbd",
|
||||
"#edbb8a",
|
||||
"#de8a5a",
|
||||
"#ca562c",
|
||||
] # based on plotly geyser
|
||||
cmap = mcolors.LinearSegmentedColormap.from_list("custom_cmap", custom_colors)
|
||||
|
||||
for _index, row in data.iterrows():
|
||||
col_normalized = (row[column] - min) / (max_value - min_value)
|
||||
color = mcolors.rgb2hex(cmap(col_normalized))
|
||||
|
||||
pnt = kml.newpoint(coords=[(row["longitude"], row["latitude"], row["height_ato"])], altitudemode=altitudemode)
|
||||
pnt.iconstyle.icon.href = "http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png"
|
||||
pnt.iconstyle.color = simplekml.Color.rgb(int(color[1:3], 16), int(color[3:5], 16), int(color[5:], 16))
|
||||
pnt.iconstyle.scale = 0.6
|
||||
pnt.description = f"Concentration: {row[column]} ppm"
|
||||
|
||||
kml.save(output_file)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Adapter wrappers — bridging current call sites to original API
|
||||
# ============================================================
|
||||
|
||||
def _scatter_3d_wrapper(df, gas):
|
||||
"""scatter_3d(df, gas) → original scatter_3d(df, color=gas_normalised)"""
|
||||
color_col = f"{gas}_normalised" if f"{gas}_normalised" in df.columns else gas
|
||||
return scatter_3d(
|
||||
df,
|
||||
color=color_col,
|
||||
colorbar_title=f"{gas.upper()} (ppm)",
|
||||
x="utm_easting",
|
||||
y="utm_northing",
|
||||
z="height_ato",
|
||||
)
|
||||
|
||||
|
||||
def _windrose_wrapper(df):
|
||||
"""windrose(df) → original windrose(df) — signature compatible"""
|
||||
return windrose(df)
|
||||
|
||||
|
||||
def _time_series_wrapper(df):
|
||||
"""time_series(df) → original time_series(df, ys=['windspeed','winddir'])"""
|
||||
return time_series(
|
||||
df,
|
||||
ys=["windspeed", "winddir"],
|
||||
x="timestamp",
|
||||
rolling_average=True,
|
||||
scatter=True,
|
||||
y_titles=["Wind Speed (m/s)", "Wind Direction (°)"],
|
||||
)
|
||||
|
||||
|
||||
def _contour_krig_wrapper(krig_variables):
|
||||
"""contour_krig(krig_variables dict) → original contour_krig(df, gas, xx, yy, field)"""
|
||||
xx = krig_variables.get("xx")
|
||||
yy = krig_variables.get("yy")
|
||||
field = krig_variables.get("field")
|
||||
gas = krig_variables.get("gas", "gas")
|
||||
|
||||
if xx is None or yy is None or field is None:
|
||||
return blank_figure()
|
||||
|
||||
# original takes df for scatter overlay — we pass an empty one
|
||||
import pandas as pd
|
||||
dummy_df = pd.DataFrame(columns=["x", "height_ato", f"{gas}_normalised"])
|
||||
|
||||
return contour_krig(dummy_df, gas, xx, yy, field, cut_ground=False)
|
||||
|
||||
|
||||
def _heatmap_krig_wrapper(krig_variables):
|
||||
"""heatmap_krig(krig_variables dict) → original heatmap_krig(xx, yy, field)"""
|
||||
xx = krig_variables.get("xx")
|
||||
yy = krig_variables.get("yy")
|
||||
field = krig_variables.get("field")
|
||||
|
||||
if xx is None or yy is None or field is None:
|
||||
return blank_figure()
|
||||
|
||||
return heatmap_krig(xx, yy, field)
|
||||
|
||||
|
||||
def _semivariogram_plot(semivariogram, gas=""):
|
||||
"""Semivariogram plot — keeps our Plotly implementation (not in original)."""
|
||||
try:
|
||||
import numpy as np
|
||||
import plotly.graph_objects as go
|
||||
|
||||
bins = semivariogram.bins
|
||||
experimental = semivariogram.experimental
|
||||
|
||||
fig = go.Figure()
|
||||
fig.add_trace(go.Scatter(x=bins, y=experimental, mode="markers",
|
||||
name="Experimental", marker={"size": 8, "color": "#3498db"}))
|
||||
|
||||
if hasattr(semivariogram, "model") and semivariogram.model is not None:
|
||||
x_line = np.linspace(0, bins.max(), 100)
|
||||
y_line = semivariogram.model(x_line)
|
||||
fig.add_trace(go.Scatter(x=x_line, y=y_line, mode="lines",
|
||||
name=f"Fitted", line={"color": "#e74c3c", "width": 2}))
|
||||
|
||||
title = "Semivariogram"
|
||||
if gas:
|
||||
title += f" – {gas.upper()}"
|
||||
fig.update_layout(title=title, xaxis_title="Lag Distance (m)",
|
||||
yaxis_title="Semivariance",
|
||||
margin={"l": 0, "r": 0, "t": 40, "b": 0})
|
||||
return fig
|
||||
except Exception:
|
||||
return blank_figure()
|
||||
|
||||
|
||||
def _outliers_wrapper(df, column, name=""):
|
||||
"""outliers(df, column, name) → original outliers(series, fence_high, fence_low)"""
|
||||
if column not in df.columns:
|
||||
return None
|
||||
valid = df[column].dropna()
|
||||
if len(valid) == 0:
|
||||
return None
|
||||
q1 = valid.quantile(0.25)
|
||||
q3 = valid.quantile(0.75)
|
||||
iqr = q3 - q1
|
||||
fence_low = q1 - 3 * iqr
|
||||
fence_high = q3 + 3 * iqr
|
||||
return outliers(valid, fence_high, fence_low)
|
||||
|
||||
|
||||
@ -129,7 +129,11 @@ def remove_outliers(df: pd.DataFrame, column: str, name: str):
|
||||
fence_low = q1 - 3 * iqr
|
||||
fence_high = q3 + 3 * iqr
|
||||
|
||||
fig = None # plotting disabled
|
||||
try:
|
||||
from . import plotting
|
||||
fig = plotting._outliers_wrapper(df, column, name)
|
||||
except Exception:
|
||||
fig = None
|
||||
|
||||
outliers = df.loc[(df[column] < fence_low) | (df[column] > fence_high)]
|
||||
if len(outliers) > 0:
|
||||
|
||||
@ -5,6 +5,7 @@ from scipy import stats
|
||||
|
||||
import pandas as pd
|
||||
import yaml
|
||||
import plotly.graph_objects as go
|
||||
|
||||
from src.gasflux import background,plotting,processing,reporting,interpolation,pre_processing,gas
|
||||
|
||||
@ -123,9 +124,22 @@ class InSituSensorStrategy(SensorStrategy):
|
||||
def process(self):
|
||||
logger.info("Processing in-situ (point) data")
|
||||
for gas in self.data_processor.gases:
|
||||
self.data_processor.figs["scatter_3d"][gas] = None
|
||||
self.data_processor.figs["windrose"] = None
|
||||
self.data_processor.figs["wind_timeseries"] = None
|
||||
# Create 3D scatter plot of flight path with gas concentration
|
||||
try:
|
||||
self.data_processor.figs["scatter_3d"][gas] = plotting._scatter_3d_wrapper(
|
||||
self.data_processor.df, gas
|
||||
)
|
||||
except Exception:
|
||||
self.data_processor.figs["scatter_3d"][gas] = plotting.blank_figure()
|
||||
# Create wind rose and time series plots
|
||||
try:
|
||||
self.data_processor.figs["windrose"] = plotting._windrose_wrapper(self.data_processor.df)
|
||||
except Exception:
|
||||
self.data_processor.figs["windrose"] = plotting.blank_figure()
|
||||
try:
|
||||
self.data_processor.figs["wind_timeseries"] = plotting._time_series_wrapper(self.data_processor.df)
|
||||
except Exception:
|
||||
self.data_processor.figs["wind_timeseries"] = plotting.blank_figure()
|
||||
|
||||
|
||||
class SpatialProcessingStrategy(ABC):
|
||||
|
||||
@ -1,11 +1,158 @@
|
||||
import requests
|
||||
import time
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 全局 Session 复用 TCP 连接,避免 SSL 握手失败
|
||||
_session = None
|
||||
|
||||
|
||||
def _get_session():
|
||||
"""获取或创建持久 HTTP Session(复用连接)。"""
|
||||
global _session
|
||||
if _session is None:
|
||||
_session = requests.Session()
|
||||
# 设置适配器,增大连接池
|
||||
from requests.adapters import HTTPAdapter
|
||||
adapter = HTTPAdapter(pool_connections=5, pool_maxsize=10, max_retries=0)
|
||||
_session.mount("https://", adapter)
|
||||
_session.mount("http://", adapter)
|
||||
return _session
|
||||
|
||||
|
||||
def _exponential_backoff_request(url, params, max_retries=4, base_timeout=45):
|
||||
"""
|
||||
带指数退避的 HTTP GET 请求。
|
||||
|
||||
Args:
|
||||
url: 请求 URL
|
||||
params: 查询参数
|
||||
max_retries: 最大重试次数(含首次尝试)
|
||||
base_timeout: 基础超时时间(秒)
|
||||
|
||||
Returns:
|
||||
requests.Response 对象
|
||||
|
||||
Raises:
|
||||
requests.RequestException: 所有重试失败后抛出
|
||||
"""
|
||||
session = _get_session()
|
||||
last_exception = None
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
# 每次重试增加超时时间(45, 60, 75, 90)
|
||||
timeout = base_timeout + attempt * 15
|
||||
response = session.get(url, params=params, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
except (requests.exceptions.ConnectionError,
|
||||
requests.exceptions.Timeout,
|
||||
requests.exceptions.SSLError) as e:
|
||||
last_exception = e
|
||||
if attempt < max_retries - 1:
|
||||
wait = 2 ** attempt # 1s, 2s, 4s, 8s
|
||||
logger.warning(
|
||||
f"Open-Meteo 请求失败 (第{attempt+1}次): {type(e).__name__}. "
|
||||
f"{wait}秒后重试... URL: {url} params: {params}"
|
||||
)
|
||||
time.sleep(wait)
|
||||
else:
|
||||
logger.error(
|
||||
f"Open-Meteo 请求最终失败 ({max_retries}次尝试): {type(e).__name__}: {e}"
|
||||
)
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
# HTTP 错误不重试(如 404, 400 等)
|
||||
logger.error(f"Open-Meteo HTTP错误: {e}")
|
||||
raise
|
||||
|
||||
raise last_exception
|
||||
|
||||
|
||||
# 缓存:相同 (lat, lon, date, time) 的请求结果,避免重复 API 调用
|
||||
# maxsize=256 足够缓存一次飞行的所有档位
|
||||
@lru_cache(maxsize=256)
|
||||
def _get_pressure_cached(lat: float, lon: float, altitude: float, date: str, time: str) -> float | None:
|
||||
"""
|
||||
带缓存的单次气压查询(内部函数)。
|
||||
坐标四舍五入到小数点后 4 位(~11m精度)以提高缓存命中率。
|
||||
"""
|
||||
# 坐标取 4 位小数作为缓存键(同一档位内坐标几乎相同)
|
||||
cache_lat = round(lat, 4)
|
||||
cache_lon = round(lon, 4)
|
||||
cache_alt = round(altitude, 1) # 高度取1位小数
|
||||
return _get_pressure_impl(cache_lat, cache_lon, cache_alt, date, time)
|
||||
|
||||
|
||||
def _get_pressure_impl(lat: float, lon: float, altitude: float, date: str, time: str) -> float | None:
|
||||
"""实际的 Open-Meteo API 调用(无缓存)。"""
|
||||
url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
params = {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"start_date": date,
|
||||
"end_date": date,
|
||||
"hourly": ["pressure_msl", "surface_pressure"],
|
||||
"timezone": "auto",
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info(f"正在获取位置 ({lat:.6f}, {lon:.6f}) 在 {date} {time} 的气压数据...")
|
||||
response = _exponential_backoff_request(url, params)
|
||||
|
||||
data = response.json()
|
||||
|
||||
if "error" in data:
|
||||
logger.error(f"Open-Meteo API错误: {data['error']}")
|
||||
return None
|
||||
|
||||
if data and "hourly" in data:
|
||||
times = data["hourly"]["time"]
|
||||
pressures = data["hourly"]["surface_pressure"]
|
||||
|
||||
if not times or not pressures:
|
||||
logger.warning("未找到气压数据(times 或 pressures 为空)")
|
||||
return None
|
||||
|
||||
target_time = f"{date}T{time}"
|
||||
if target_time in times:
|
||||
idx = times.index(target_time)
|
||||
pressure = pressures[idx]
|
||||
logger.info(f"成功获取气压: {pressure} hPa")
|
||||
return pressure
|
||||
else:
|
||||
logger.warning(f"在数据中未找到时间: {target_time},可用范围: {times[0]} 到 {times[-1]}")
|
||||
# 回退:取最近的小时
|
||||
try:
|
||||
closest = min(times, key=lambda t: abs(
|
||||
(int(t.split("T")[1].split(":")[0]) if "T" in t else 0) -
|
||||
(int(time.split(":")[0]))
|
||||
))
|
||||
idx = times.index(closest)
|
||||
pressure = pressures[idx]
|
||||
logger.info(f"使用最近时间 {closest} 的气压: {pressure} hPa")
|
||||
return pressure
|
||||
except Exception:
|
||||
return None
|
||||
else:
|
||||
logger.warning("API响应中没有hourly数据")
|
||||
return None
|
||||
|
||||
except requests.exceptions.HTTPError:
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"获取气压时发生未知错误: {type(e).__name__}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_pressure_at_location(lat, lon, altitude, date, time, max_retries=3, timeout=30):
|
||||
"""
|
||||
获取指定位置、时间、高度的气压
|
||||
获取指定位置、时间、高度的气压(公共接口,保持向后兼容)。
|
||||
|
||||
Args:
|
||||
lat: 纬度
|
||||
@ -13,8 +160,8 @@ def get_pressure_at_location(lat, lon, altitude, date, time, max_retries=3, time
|
||||
altitude: 海拔高度 (米)
|
||||
date: 日期 (格式: YYYY-MM-DD 或 YYYY/MM/DD)
|
||||
time: 时间 (格式: HH:MM 或 HH:MM:SS)
|
||||
max_retries: 最大重试次数
|
||||
timeout: 请求超时时间(秒)
|
||||
max_retries: 最大重试次数(已废弃,由内部指数退避处理)
|
||||
timeout: 请求超时时间(秒)(已废弃,由内部控制)
|
||||
|
||||
Returns:
|
||||
float: 气压值 (hPa),获取失败返回 None
|
||||
@ -22,130 +169,42 @@ def get_pressure_at_location(lat, lon, altitude, date, time, max_retries=3, time
|
||||
|
||||
# 标准化日期格式为 YYYY-MM-DD
|
||||
def normalize_date(d):
|
||||
"""将各种日期格式标准化为 YYYY-MM-DD"""
|
||||
if not d:
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
# 处理斜杠分隔符
|
||||
if "/" in d:
|
||||
d = d.replace("/", "-")
|
||||
|
||||
parts = d.split("-")
|
||||
if len(parts) == 3:
|
||||
year = parts[0]
|
||||
month = parts[1].zfill(2) # 确保月份是两位数
|
||||
day = parts[2].zfill(2) # 确保日期是两位数
|
||||
return f"{year}-{month}-{day}"
|
||||
else:
|
||||
# 如果格式不正确,返回今天的日期
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
return f"{parts[0]}-{parts[1].zfill(2)}-{parts[2].zfill(2)}"
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
date = normalize_date(date)
|
||||
|
||||
# 标准化时间格式为 HH:MM
|
||||
def normalize_time(t):
|
||||
"""将各种时间格式标准化为 HH:MM"""
|
||||
if not t or ":" not in t:
|
||||
return "12:00" # 默认中午12点
|
||||
|
||||
return "12:00"
|
||||
parts = t.split(":")
|
||||
if len(parts) >= 2:
|
||||
hour = parts[0].zfill(2) # 确保小时是两位数
|
||||
minute = parts[1].zfill(2) # 确保分钟是两位数
|
||||
return f"{hour}:{minute}"
|
||||
elif len(parts) == 1:
|
||||
hour = parts[0].zfill(2)
|
||||
return f"{hour}:00"
|
||||
else:
|
||||
return "12:00"
|
||||
return f"{parts[0].zfill(2)}:{parts[1].zfill(2)}"
|
||||
return "12:00"
|
||||
|
||||
time = normalize_time(time)
|
||||
|
||||
url = "https://archive-api.open-meteo.com/v1/archive"
|
||||
|
||||
params = {
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"start_date": date, # 格式: YYYY-MM-DD
|
||||
"end_date": date,
|
||||
"hourly": ["pressure_msl", "surface_pressure"],
|
||||
"timezone": "auto"
|
||||
}
|
||||
|
||||
# 创建带有重试机制的会话
|
||||
session = requests.Session()
|
||||
retry_strategy = Retry(
|
||||
total=max_retries,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
backoff_factor=1
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
try:
|
||||
print(f"正在获取位置 ({lat:.6f}, {lon:.6f}) 在 {date} {time} 的气压数据...")
|
||||
response = session.get(url, params=params, timeout=timeout)
|
||||
|
||||
# 检查响应状态
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
# 检查API错误
|
||||
if "error" in data:
|
||||
print(f"API错误: {data['error']}")
|
||||
return None
|
||||
|
||||
# 解析气压数据
|
||||
if data and "hourly" in data:
|
||||
times = data["hourly"]["time"]
|
||||
pressures = data["hourly"]["surface_pressure"] # 地表气压
|
||||
|
||||
if not times or not pressures:
|
||||
print("未找到气压数据")
|
||||
return None
|
||||
|
||||
# 根据时间找到对应气压
|
||||
target_time = f"{date}T{time}"
|
||||
if target_time in times:
|
||||
idx = times.index(target_time)
|
||||
pressure = pressures[idx]
|
||||
print(f"成功获取气压: {pressure} hPa")
|
||||
return pressure
|
||||
else:
|
||||
print(f"在数据中未找到时间: {target_time}")
|
||||
print(f"可用时间范围: {times[0]} 到 {times[-1]}")
|
||||
return None
|
||||
else:
|
||||
print("API响应中没有hourly数据")
|
||||
return None
|
||||
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
print(f"网络连接错误: {e}")
|
||||
print("请检查网络连接或稍后重试")
|
||||
return None
|
||||
except requests.exceptions.Timeout as e:
|
||||
print(f"请求超时: {e}")
|
||||
print(f"已重试 {max_retries} 次,请检查网络连接")
|
||||
return None
|
||||
except requests.exceptions.HTTPError as e:
|
||||
print(f"HTTP错误: {e}")
|
||||
return None
|
||||
except ValueError as e:
|
||||
print(f"数据解析错误: {e}")
|
||||
return None
|
||||
return _get_pressure_cached(
|
||||
float(lat), float(lon), float(altitude), date, time
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"未知错误: {e}")
|
||||
logger.error(f"get_pressure_at_location 失败: {type(e).__name__}: {e}")
|
||||
return None
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def batch_get_pressure(data_list):
|
||||
"""
|
||||
批量获取多个位置的气压数据
|
||||
批量获取多个位置的气压数据。
|
||||
|
||||
Args:
|
||||
data_list: 包含 (lat, lon, altitude, date, time) 元组的列表
|
||||
@ -154,37 +213,36 @@ def batch_get_pressure(data_list):
|
||||
list: 气压值列表
|
||||
"""
|
||||
results = []
|
||||
for i, (lat, lon, alt, date, time) in enumerate(data_list):
|
||||
for i, (lat, lon, alt, date, time_val) in enumerate(data_list):
|
||||
print(f"\n处理第 {i+1} 个位置...")
|
||||
pressure = get_pressure_at_location(lat, lon, alt, date, time)
|
||||
pressure = get_pressure_at_location(lat, lon, alt, date, time_val)
|
||||
results.append(pressure)
|
||||
if pressure is not None:
|
||||
print(f"位置 {i+1}: {pressure} hPa")
|
||||
else:
|
||||
print(f"位置 {i+1}: 获取失败")
|
||||
|
||||
# 添加短暂延迟,避免请求过于频繁
|
||||
# 短暂延迟避免请求过于频繁
|
||||
if i < len(data_list) - 1:
|
||||
time.sleep(0.5)
|
||||
time.sleep(0.3)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# 使用示例
|
||||
if __name__ == "__main__":
|
||||
print("=== 气压数据获取工具 ===\n")
|
||||
|
||||
# 单个位置示例
|
||||
print("1. 单个位置查询:")
|
||||
pressure = get_pressure_at_location(
|
||||
lat=40.3491370, # 纽约纬度
|
||||
lon=115.7855289, # 纽约经度 (西经)
|
||||
altitude=435.789, # 海拔10米
|
||||
lat=40.3491370,
|
||||
lon=115.7855289,
|
||||
altitude=435.789,
|
||||
date="2016-02-12",
|
||||
time="08:00" # HH:MM格式
|
||||
time="08:00"
|
||||
)
|
||||
|
||||
if pressure is not None:
|
||||
print(f"纽约当前气压: {pressure} hPa")
|
||||
else:
|
||||
print("获取纽约气压数据失败")
|
||||
|
||||
|
||||
@ -27,13 +27,22 @@ def mass_balance_report(
|
||||
"""Generate a mass balance report (plots disabled)."""
|
||||
template_path = Path(__file__).parent / "templates" / "mass_balance_template.html"
|
||||
|
||||
# Plots disabled -> use empty strings
|
||||
# Convert figures to HTML strings
|
||||
def _fig_to_html(fig) -> str:
|
||||
"""Safely convert a plotly figure to HTML string."""
|
||||
if fig is None:
|
||||
return ""
|
||||
try:
|
||||
return fig.to_html(full_html=False, include_plotlyjs=True)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
plot_htmls = {
|
||||
"3D": "",
|
||||
"krig": "",
|
||||
"windrose": "",
|
||||
"wind": "",
|
||||
"background": "",
|
||||
"3D": _fig_to_html(threed_fig),
|
||||
"krig": _fig_to_html(krig_fig),
|
||||
"windrose": _fig_to_html(windrose_fig),
|
||||
"wind": _fig_to_html(wind_fig),
|
||||
"background": _fig_to_html(background_fig),
|
||||
}
|
||||
|
||||
summary_data = {
|
||||
|
||||
Reference in New Issue
Block a user