Compare commits
11 Commits
408e1660fc
...
1.2
| Author | SHA1 | Date | |
|---|---|---|---|
| c6b9783a7a | |||
| 55cb7fb025 | |||
| adaabb2831 | |||
| 9bafc9ba0e | |||
| 9898b68410 | |||
| fa0cd769a8 | |||
| 3367774574 | |||
| 616783fb52 | |||
| 4ec46208dd | |||
| 5609a8d708 | |||
| e5f43f0297 |
@ -5,19 +5,19 @@ REM This script builds a standalone executable using Waitress WSGI server
|
|||||||
echo Building GasFlux Web API executable...
|
echo Building GasFlux Web API executable...
|
||||||
|
|
||||||
REM Check if PyInstaller is installed
|
REM Check if PyInstaller is installed
|
||||||
python -c "import PyInstaller" >nul 2>&1
|
py -3 -c "import PyInstaller" >nul 2>&1
|
||||||
if errorlevel 1 (
|
if errorlevel 1 (
|
||||||
echo Error: PyInstaller is not installed. Please run:
|
echo Error: PyInstaller is not installed. Please run:
|
||||||
echo pip install pyinstaller waitress
|
echo py -3 -m pip install pyinstaller waitress
|
||||||
pause
|
pause
|
||||||
exit /b 1
|
exit /b 1
|
||||||
)
|
)
|
||||||
|
|
||||||
REM Check if Waitress is installed
|
REM Check if Waitress is installed
|
||||||
python -c "import waitress" >nul 2>&1
|
py -3 -c "import waitress" >nul 2>&1
|
||||||
if errorlevel 1 (
|
if errorlevel 1 (
|
||||||
echo Error: Waitress is not installed. Please run:
|
echo Error: Waitress is not installed. Please run:
|
||||||
echo pip install waitress
|
echo py -3 -m pip install waitress
|
||||||
pause
|
pause
|
||||||
exit /b 1
|
exit /b 1
|
||||||
)
|
)
|
||||||
@ -41,8 +41,24 @@ pyinstaller --onefile ^
|
|||||||
--hidden-import skgstat ^
|
--hidden-import skgstat ^
|
||||||
--hidden-import skgstat.Variogram ^
|
--hidden-import skgstat.Variogram ^
|
||||||
--hidden-import skgstat.OrdinaryKriging ^
|
--hidden-import skgstat.OrdinaryKriging ^
|
||||||
--add-data "src\gasflux\gasflux_config.yaml;src\gasflux" ^
|
--hidden-import skgstat.DirectionalVariogram ^
|
||||||
--add-data "API_DOCUMENTATION.md;." ^
|
--hidden-import plotly ^
|
||||||
|
--hidden-import plotly.graph_objects ^
|
||||||
|
--hidden-import plotly.express ^
|
||||||
|
--hidden-import plotly.subplots ^
|
||||||
|
--hidden-import pybaselines ^
|
||||||
|
--hidden-import scipy ^
|
||||||
|
--hidden-import scipy.odr ^
|
||||||
|
--hidden-import scipy.signal ^
|
||||||
|
--hidden-import scipy.stats ^
|
||||||
|
--hidden-import scipy.integrate ^
|
||||||
|
--hidden-import scipy.optimize ^
|
||||||
|
--hidden-import molmass ^
|
||||||
|
--hidden-import openpyxl ^
|
||||||
|
--hidden-import pyproj ^
|
||||||
|
--hidden-import jinja2 ^
|
||||||
|
--hidden-import requests ^
|
||||||
|
--hidden-import urllib3 ^
|
||||||
--hidden-import matplotlib ^
|
--hidden-import matplotlib ^
|
||||||
--hidden-import matplotlib.pyplot ^
|
--hidden-import matplotlib.pyplot ^
|
||||||
--hidden-import matplotlib.backends ^
|
--hidden-import matplotlib.backends ^
|
||||||
@ -53,7 +69,19 @@ pyinstaller --onefile ^
|
|||||||
--hidden-import matplotlib.patches ^
|
--hidden-import matplotlib.patches ^
|
||||||
--hidden-import matplotlib.text ^
|
--hidden-import matplotlib.text ^
|
||||||
--hidden-import matplotlib.transforms ^
|
--hidden-import matplotlib.transforms ^
|
||||||
|
--hidden-import tqdm ^
|
||||||
|
--hidden-import certifi ^
|
||||||
|
--hidden-import charset_normalizer ^
|
||||||
|
--hidden-import geopandas ^
|
||||||
|
--hidden-import shapely ^
|
||||||
|
--hidden-import fiona ^
|
||||||
|
--hidden-import simplekml ^
|
||||||
|
--hidden-import joblib ^
|
||||||
--exclude-module tkinter ^
|
--exclude-module tkinter ^
|
||||||
|
--add-data "src\gasflux\gasflux_config.yaml;src\gasflux" ^
|
||||||
|
--add-data "src\gasflux\templates\mass_balance_template.html;src\gasflux\templates" ^
|
||||||
|
--add-data "gasflux.ini;." ^
|
||||||
|
--add-data "gasflux.ini.example;." ^
|
||||||
server_waitress.py
|
server_waitress.py
|
||||||
|
|
||||||
if errorlevel 1 (
|
if errorlevel 1 (
|
||||||
@ -69,6 +97,8 @@ echo.
|
|||||||
echo To run the server:
|
echo To run the server:
|
||||||
echo GasFluxAPI.exe
|
echo GasFluxAPI.exe
|
||||||
echo.
|
echo.
|
||||||
echo The server will start on http://localhost:5000
|
echo The server will start on http://localhost:5001
|
||||||
|
echo Make sure gasflux.ini is in the same directory as the executable,
|
||||||
|
echo or configure paths via environment variables.
|
||||||
echo.
|
echo.
|
||||||
pause
|
pause
|
||||||
@ -30,9 +30,9 @@ task_cleanup_interval = 30
|
|||||||
# 24 hours in seconds
|
# 24 hours in seconds
|
||||||
max_task_age = 86400
|
max_task_age = 86400
|
||||||
# 1 minute in seconds for successful tasks
|
# 1 minute in seconds for successful tasks
|
||||||
successful_task_cleanup_age = 60
|
successful_task_cleanup_age = 315360000
|
||||||
# 1 minute in seconds for failed tasks
|
# 1 minute in seconds for failed tasks
|
||||||
failed_task_cleanup_age = 60
|
failed_task_cleanup_age = 315360000
|
||||||
janitor_dry_run = false
|
janitor_dry_run = false
|
||||||
|
|
||||||
[performance]
|
[performance]
|
||||||
|
|||||||
@ -45,9 +45,17 @@ import sys
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# 设置控制台编码为 UTF-8(解决 Windows 中文乱码问题)
|
# 设置控制台编码为 UTF-8(解决 Windows 中文乱码问题)
|
||||||
|
# PyInstaller onefile 模式下 sys.stdout.buffer 可能不存在(无控制台或管道重定向),
|
||||||
|
# 此时跳过编码包装,避免 AttributeError 导致启动崩溃
|
||||||
import io
|
import io
|
||||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
try:
|
||||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||||
|
except (AttributeError, ValueError):
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||||
|
except (AttributeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
# Add the project root and src directory to PYTHONPATH
|
# Add the project root and src directory to PYTHONPATH
|
||||||
project_root = Path(__file__).parent.absolute()
|
project_root = Path(__file__).parent.absolute()
|
||||||
|
|||||||
@ -3,6 +3,7 @@ Download Blueprint
|
|||||||
Handles file download endpoints.
|
Handles file download endpoints.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from flask import Blueprint, send_file, current_app
|
from flask import Blueprint, send_file, current_app
|
||||||
|
|
||||||
@ -10,40 +11,6 @@ from ..shared import _format_response, log_performance, logger
|
|||||||
from ..auth import require_api_key
|
from ..auth import require_api_key
|
||||||
|
|
||||||
|
|
||||||
def _mark_task_downloaded(task_id):
|
|
||||||
"""Mark task as downloaded and schedule deletion in database."""
|
|
||||||
import sqlite3
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
# Use independent database connection (not from flask.g which may be closed)
|
|
||||||
from ..db import get_db_path as get_config_db_path
|
|
||||||
db_path = get_config_db_path(current_app)
|
|
||||||
|
|
||||||
# Get cleanup age for successful tasks from config (in seconds)
|
|
||||||
successful_task_cleanup_age = current_app.config.get('SUCCESSFUL_TASK_CLEANUP_AGE', 3600)
|
|
||||||
|
|
||||||
try:
|
|
||||||
conn = sqlite3.connect(str(db_path), check_same_thread=False)
|
|
||||||
conn.execute("PRAGMA foreign_keys=ON")
|
|
||||||
conn.execute("PRAGMA busy_timeout=3000")
|
|
||||||
|
|
||||||
# Update downloaded timestamp and set deletion time based on config
|
|
||||||
conn.execute("""
|
|
||||||
UPDATE tasks
|
|
||||||
SET downloaded_at = datetime('now', '+8 hours'),
|
|
||||||
delete_after_at = datetime('now', '+8 hours', '+' || ? || ' seconds')
|
|
||||||
WHERE task_id = ?
|
|
||||||
""", (successful_task_cleanup_age, task_id))
|
|
||||||
|
|
||||||
conn.commit()
|
|
||||||
logger.info(f"Task {task_id} marked as downloaded, scheduled for deletion in {successful_task_cleanup_age} seconds")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to mark task {task_id} as downloaded: {str(e)}", exc_info=True)
|
|
||||||
finally:
|
|
||||||
if 'conn' in locals():
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
# Create blueprint
|
# Create blueprint
|
||||||
download_bp = Blueprint('download', __name__, url_prefix='/download')
|
download_bp = Blueprint('download', __name__, url_prefix='/download')
|
||||||
|
|
||||||
@ -124,12 +91,19 @@ def download_file(filename):
|
|||||||
file_size = file_path.stat().st_size
|
file_size = file_path.stat().st_size
|
||||||
logger.info(f"Serving file: {filename} ({file_size} bytes)")
|
logger.info(f"Serving file: {filename} ({file_size} bytes)")
|
||||||
|
|
||||||
# Mark download immediately before sending file
|
# 记录下载时间(仅时间戳,不设置自动删除)
|
||||||
if task_id:
|
if task_id:
|
||||||
try:
|
try:
|
||||||
_mark_task_downloaded(task_id)
|
from ..db import get_db
|
||||||
|
db = get_db()
|
||||||
|
db.execute(
|
||||||
|
"UPDATE tasks SET downloaded_at = datetime('now', '+8 hours') WHERE task_id = ?",
|
||||||
|
(task_id,)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
logger.info(f"Task {task_id} downloaded at {datetime.now()}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to mark download for task {task_id}: {str(e)}")
|
logger.error(f"Failed to record download for task {task_id}: {str(e)}")
|
||||||
|
|
||||||
response = send_file(file_path)
|
response = send_file(file_path)
|
||||||
return response
|
return response
|
||||||
|
|||||||
@ -99,11 +99,6 @@ def ordinary_kriging(
|
|||||||
# np.nan_to_num(error_1s, copy=False, nan=0)
|
# np.nan_to_num(error_1s, copy=False, nan=0)
|
||||||
volume_error = simpsonintegrate(error_1s, x_cell_size, y_cell_size)
|
volume_error = simpsonintegrate(error_1s, x_cell_size, y_cell_size)
|
||||||
|
|
||||||
# Plots disabled
|
|
||||||
contour_plot = None
|
|
||||||
grid_plot = None
|
|
||||||
semivariogram_plot = None
|
|
||||||
|
|
||||||
output_text = (
|
output_text = (
|
||||||
f"The emissions flux of {gas.upper()} is {volume:.3f}kgh⁻¹; "
|
f"The emissions flux of {gas.upper()} is {volume:.3f}kgh⁻¹; "
|
||||||
f"the cut and fill volumes of the grid are {volumepos:.3f} and {volumeneg:.3f}kgh⁻¹. "
|
f"the cut and fill volumes of the grid are {volumepos:.3f} and {volumeneg:.3f}kgh⁻¹. "
|
||||||
@ -123,6 +118,20 @@ def ordinary_kriging(
|
|||||||
"volume_error": volume_error,
|
"volume_error": volume_error,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Generate plots (must be after krig_variables is defined)
|
||||||
|
try:
|
||||||
|
contour_plot = plotting._contour_krig_wrapper(krig_variables)
|
||||||
|
except Exception:
|
||||||
|
contour_plot = None
|
||||||
|
try:
|
||||||
|
grid_plot = plotting._heatmap_krig_wrapper(krig_variables)
|
||||||
|
except Exception:
|
||||||
|
grid_plot = None
|
||||||
|
try:
|
||||||
|
semivariogram_plot = plotting._semivariogram_plot(semivariogram, gas)
|
||||||
|
except Exception:
|
||||||
|
semivariogram_plot = None
|
||||||
|
|
||||||
return krig_variables, output_text, contour_plot, grid_plot, semivariogram_plot
|
return krig_variables, output_text, contour_plot, grid_plot, semivariogram_plot
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -310,8 +310,7 @@ def reconcile_tasks_on_startup():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
current_app.logger.error(f"Failed to delete failed task {task_id}: {str(e)}", exc_info=True)
|
current_app.logger.error(f"Failed to delete failed task {task_id}: {str(e)}", exc_info=True)
|
||||||
|
|
||||||
# Check for tasks with output directories that no longer exist
|
# 检查 output_dir 不存在的任务(仅警告,不自动删除)
|
||||||
# This helps clean up database entries for manually deleted directories
|
|
||||||
rows = conn.execute("""
|
rows = conn.execute("""
|
||||||
SELECT task_id, output_dir
|
SELECT task_id, output_dir
|
||||||
FROM tasks
|
FROM tasks
|
||||||
@ -321,18 +320,21 @@ def reconcile_tasks_on_startup():
|
|||||||
|
|
||||||
orphaned_count = 0
|
orphaned_count = 0
|
||||||
for row in rows:
|
for row in rows:
|
||||||
task_id_from_db = row[0] # task_id是第一个字段
|
task_id_from_db = row[0]
|
||||||
output_dir_from_db = row[1] # output_dir是第二个字段
|
output_dir_from_db = row[1]
|
||||||
|
|
||||||
if not Path(output_dir_from_db).exists():
|
if not Path(output_dir_from_db).exists():
|
||||||
conn.execute(
|
current_app.logger.warning(
|
||||||
"DELETE FROM tasks WHERE task_id = ?",
|
f"Startup reconciliation: Task {task_id_from_db} output directory "
|
||||||
(task_id_from_db,)
|
f"not found on disk: {output_dir_from_db}. Database record preserved."
|
||||||
)
|
)
|
||||||
orphaned_count += 1
|
orphaned_count += 1
|
||||||
|
|
||||||
if orphaned_count > 0:
|
if orphaned_count > 0:
|
||||||
current_app.logger.info(f"Startup reconciliation: Marked {orphaned_count} tasks as deleted (directories not found)")
|
current_app.logger.info(
|
||||||
|
f"Startup reconciliation: Found {orphaned_count} tasks with missing directories "
|
||||||
|
f"(records preserved, use DELETE /task/<id> to manually clean up)"
|
||||||
|
)
|
||||||
|
|
||||||
# Backfill output_dir for rows with NULL, using OUTPUT_FOLDER/<task_id>
|
# Backfill output_dir for rows with NULL, using OUTPUT_FOLDER/<task_id>
|
||||||
output_base = Path(current_app.config.get('OUTPUT_FOLDER') or '')
|
output_base = Path(current_app.config.get('OUTPUT_FOLDER') or '')
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@ -129,7 +129,11 @@ def remove_outliers(df: pd.DataFrame, column: str, name: str):
|
|||||||
fence_low = q1 - 3 * iqr
|
fence_low = q1 - 3 * iqr
|
||||||
fence_high = q3 + 3 * iqr
|
fence_high = q3 + 3 * iqr
|
||||||
|
|
||||||
fig = None # plotting disabled
|
try:
|
||||||
|
from . import plotting
|
||||||
|
fig = plotting._outliers_wrapper(df, column, name)
|
||||||
|
except Exception:
|
||||||
|
fig = None
|
||||||
|
|
||||||
outliers = df.loc[(df[column] < fence_low) | (df[column] > fence_high)]
|
outliers = df.loc[(df[column] < fence_low) | (df[column] > fence_high)]
|
||||||
if len(outliers) > 0:
|
if len(outliers) > 0:
|
||||||
|
|||||||
@ -5,6 +5,7 @@ from scipy import stats
|
|||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import yaml
|
import yaml
|
||||||
|
import plotly.graph_objects as go
|
||||||
|
|
||||||
from src.gasflux import background,plotting,processing,reporting,interpolation,pre_processing,gas
|
from src.gasflux import background,plotting,processing,reporting,interpolation,pre_processing,gas
|
||||||
|
|
||||||
@ -123,9 +124,22 @@ class InSituSensorStrategy(SensorStrategy):
|
|||||||
def process(self):
|
def process(self):
|
||||||
logger.info("Processing in-situ (point) data")
|
logger.info("Processing in-situ (point) data")
|
||||||
for gas in self.data_processor.gases:
|
for gas in self.data_processor.gases:
|
||||||
self.data_processor.figs["scatter_3d"][gas] = None
|
# Create 3D scatter plot of flight path with gas concentration
|
||||||
self.data_processor.figs["windrose"] = None
|
try:
|
||||||
self.data_processor.figs["wind_timeseries"] = None
|
self.data_processor.figs["scatter_3d"][gas] = plotting._scatter_3d_wrapper(
|
||||||
|
self.data_processor.df, gas
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
self.data_processor.figs["scatter_3d"][gas] = plotting.blank_figure()
|
||||||
|
# Create wind rose and time series plots
|
||||||
|
try:
|
||||||
|
self.data_processor.figs["windrose"] = plotting._windrose_wrapper(self.data_processor.df)
|
||||||
|
except Exception:
|
||||||
|
self.data_processor.figs["windrose"] = plotting.blank_figure()
|
||||||
|
try:
|
||||||
|
self.data_processor.figs["wind_timeseries"] = plotting._time_series_wrapper(self.data_processor.df)
|
||||||
|
except Exception:
|
||||||
|
self.data_processor.figs["wind_timeseries"] = plotting.blank_figure()
|
||||||
|
|
||||||
|
|
||||||
class SpatialProcessingStrategy(ABC):
|
class SpatialProcessingStrategy(ABC):
|
||||||
|
|||||||
@ -27,13 +27,22 @@ def mass_balance_report(
|
|||||||
"""Generate a mass balance report (plots disabled)."""
|
"""Generate a mass balance report (plots disabled)."""
|
||||||
template_path = Path(__file__).parent / "templates" / "mass_balance_template.html"
|
template_path = Path(__file__).parent / "templates" / "mass_balance_template.html"
|
||||||
|
|
||||||
# Plots disabled -> use empty strings
|
# Convert figures to HTML strings
|
||||||
|
def _fig_to_html(fig) -> str:
|
||||||
|
"""Safely convert a plotly figure to HTML string."""
|
||||||
|
if fig is None:
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
return fig.to_html(full_html=False, include_plotlyjs=True)
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
plot_htmls = {
|
plot_htmls = {
|
||||||
"3D": "",
|
"3D": _fig_to_html(threed_fig),
|
||||||
"krig": "",
|
"krig": _fig_to_html(krig_fig),
|
||||||
"windrose": "",
|
"windrose": _fig_to_html(windrose_fig),
|
||||||
"wind": "",
|
"wind": _fig_to_html(wind_fig),
|
||||||
"background": "",
|
"background": _fig_to_html(background_fig),
|
||||||
}
|
}
|
||||||
|
|
||||||
summary_data = {
|
summary_data = {
|
||||||
|
|||||||
Reference in New Issue
Block a user