重构: 切换存储至SQLite，启用INI配置与API Key校验
This commit is contained in:
@ -15,45 +15,64 @@ import yaml
|
||||
# Shared utilities imported from shared.py
|
||||
try:
|
||||
# Try relative import (when run as part of package)
|
||||
from .shared import task_status, TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED
|
||||
from .shared import TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED, update_task_status as shared_update_task_status
|
||||
except ImportError:
|
||||
# Fallback to absolute import (when run directly)
|
||||
from shared import task_status, TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED
|
||||
from shared import TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED, update_task_status as shared_update_task_status
|
||||
|
||||
# Load configuration from INI file
|
||||
try:
|
||||
from .config_reader import config_reader
|
||||
except ImportError:
|
||||
from config_reader import config_reader
|
||||
|
||||
# Blueprints will be imported after app initialization to avoid circular imports
|
||||
|
||||
# Environment-based configuration management
|
||||
class Config:
|
||||
"""Configuration management using environment variables with defaults."""
|
||||
"""Configuration management using INI file with environment variable fallbacks."""
|
||||
|
||||
# Server configuration
|
||||
HOST = os.getenv('GASFLUX_HOST', '0.0.0.0')
|
||||
PORT = int(os.getenv('GASFLUX_PORT', '5000'))
|
||||
DEBUG = os.getenv('GASFLUX_DEBUG', 'false').lower() in ('true', '1', 'yes', 'on')
|
||||
# Server configuration from config_reader
|
||||
HOST = config_reader.host
|
||||
PORT = config_reader.port
|
||||
DEBUG = config_reader.debug
|
||||
BASE_URL = config_reader.base_url
|
||||
|
||||
# Directory configuration
|
||||
BASE_DIR = None # Will be set dynamically
|
||||
UPLOAD_FOLDER_NAME = os.getenv('GASFLUX_UPLOAD_FOLDER', 'web_api_data/uploads')
|
||||
OUTPUT_FOLDER_NAME = os.getenv('GASFLUX_OUTPUT_FOLDER', 'web_api_data/outputs')
|
||||
UPLOAD_FOLDER_NAME = str(config_reader.uploads_path)
|
||||
OUTPUT_FOLDER_NAME = str(config_reader.outputs_path)
|
||||
|
||||
# File size limits (in bytes)
|
||||
MAX_CONTENT_LENGTH = int(os.getenv('GASFLUX_MAX_CONTENT_LENGTH', str(100 * 1024 * 1024))) # 100MB
|
||||
MAX_CONTENT_LENGTH = config_reader.max_content_length
|
||||
|
||||
# Logging configuration
|
||||
LOG_LEVEL = os.getenv('GASFLUX_LOG_LEVEL', 'INFO').upper()
|
||||
LOG_FILE = os.getenv('GASFLUX_LOG_FILE', 'logs/gasflux_api.log')
|
||||
LOG_LEVEL = config_reader.log_level.upper()
|
||||
LOG_FILE = config_reader.log_file
|
||||
|
||||
# CORS configuration
|
||||
# CORS configuration (keeping environment fallback for now)
|
||||
CORS_ORIGINS = os.getenv('GASFLUX_CORS_ORIGINS', '*').split(',')
|
||||
|
||||
# Task management
|
||||
TASK_CLEANUP_INTERVAL = int(os.getenv('GASFLUX_TASK_CLEANUP_INTERVAL', '3600')) # 1 hour in seconds
|
||||
MAX_TASK_AGE = int(os.getenv('GASFLUX_MAX_TASK_AGE', str(24 * 3600))) # 24 hours in seconds
|
||||
TASK_CLEANUP_INTERVAL = config_reader.task_cleanup_interval
|
||||
MAX_TASK_AGE = config_reader.max_task_age
|
||||
|
||||
# Performance tuning
|
||||
THREADS = int(os.getenv('GASFLUX_THREADS', '8')) # Waitress threads
|
||||
CONNECTION_LIMIT = int(os.getenv('GASFLUX_CONNECTION_LIMIT', '100'))
|
||||
CHANNEL_TIMEOUT = int(os.getenv('GASFLUX_CHANNEL_TIMEOUT', '300')) # 5 minutes
|
||||
THREADS = config_reader.threads
|
||||
CONNECTION_LIMIT = config_reader.connection_limit
|
||||
CHANNEL_TIMEOUT = config_reader.channel_timeout
|
||||
|
||||
# Database configuration
|
||||
DB_PATH = config_reader.db_path if config_reader.db_path else None
|
||||
|
||||
# Persistence backend
|
||||
TASK_PERSIST_BACKEND = config_reader.persist_backend
|
||||
|
||||
# Janitor configuration
|
||||
JANITOR_DRY_RUN = config_reader.janitor_dry_run
|
||||
|
||||
# Admin bootstrap key
|
||||
ADMIN_BOOTSTRAP_KEY = config_reader.admin_bootstrap_key
|
||||
|
||||
@classmethod
|
||||
def init_base_dir(cls):
|
||||
@ -74,40 +93,35 @@ class Config:
|
||||
|
||||
@classmethod
|
||||
def init_directories(cls, output_dir=None):
|
||||
"""Initialize upload and output directories."""
|
||||
if output_dir:
|
||||
# Use config-based output directory
|
||||
output_base = Path(output_dir)
|
||||
if not output_base.is_absolute():
|
||||
output_base = cls.BASE_DIR / output_base
|
||||
else:
|
||||
# Use default relative paths
|
||||
output_base = cls.BASE_DIR
|
||||
"""Initialize upload and output directories from configuration."""
|
||||
# Use paths from config_reader
|
||||
uploads_path = config_reader.uploads_path
|
||||
outputs_path = config_reader.outputs_path
|
||||
|
||||
# Set upload and output directories relative to output_base
|
||||
cls.UPLOAD_FOLDER = output_base / "uploads"
|
||||
cls.OUTPUT_FOLDER = output_base / "outputs"
|
||||
# Resolve relative paths to absolute if needed
|
||||
if not uploads_path.is_absolute():
|
||||
uploads_path = cls.BASE_DIR / uploads_path
|
||||
if not outputs_path.is_absolute():
|
||||
outputs_path = cls.BASE_DIR / outputs_path
|
||||
|
||||
# Set the resolved paths
|
||||
cls.UPLOAD_FOLDER = uploads_path
|
||||
cls.OUTPUT_FOLDER = outputs_path
|
||||
|
||||
# Create directories
|
||||
cls.UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
cls.OUTPUT_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Directories initialized - Upload: {cls.UPLOAD_FOLDER}, Output: {cls.OUTPUT_FOLDER}")
|
||||
|
||||
# For backward compatibility, also set the old-style paths
|
||||
if output_dir:
|
||||
logger.warning("output_dir parameter is deprecated, use gasflux.ini [paths] section instead")
|
||||
|
||||
@classmethod
|
||||
def update_directories_from_config(cls, config_path=None):
|
||||
"""Update directories based on config file."""
|
||||
if config_path and Path(config_path).exists():
|
||||
try:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
config = yaml.safe_load(f)
|
||||
output_dir = config.get('output_dir')
|
||||
if output_dir:
|
||||
cls.init_directories(output_dir)
|
||||
logger.info(f"Directories updated from config: {config_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update directories from config {config_path}: {e}")
|
||||
else:
|
||||
logger.info("Using default directory configuration")
|
||||
"""Update directories based on config file. (DEPRECATED: Use gasflux.ini instead)"""
|
||||
logger.warning("update_directories_from_config is deprecated. Output directories are now configured via gasflux.ini [paths] section.")
|
||||
# No longer reads output_dir from YAML config - directories are set from INI config in init_directories()
|
||||
|
||||
@classmethod
|
||||
def get_log_level(cls):
|
||||
@ -175,33 +189,9 @@ def log_performance(func):
|
||||
# Task status management
|
||||
# Task status constants and storage moved to shared.py
|
||||
|
||||
def update_task_status(task_id, status, message=None, results=None, error=None):
|
||||
"""Update task status in the global dictionary."""
|
||||
timestamp = time.time()
|
||||
old_status = task_status.get(task_id, {}).get("status", "unknown")
|
||||
|
||||
task_status[task_id] = {
|
||||
"status": status,
|
||||
"message": message,
|
||||
"results": results,
|
||||
"error": error,
|
||||
"updated_at": timestamp
|
||||
}
|
||||
|
||||
# Log detailed status change with context
|
||||
log_msg = f"Task {task_id} status changed: {old_status} -> {status}"
|
||||
if message:
|
||||
log_msg += f" | Message: {message}"
|
||||
if results:
|
||||
log_msg += f" | Results count: {len(results) if isinstance(results, list) else 'N/A'}"
|
||||
if error:
|
||||
log_msg += f" | Error: {error}"
|
||||
|
||||
log_level = logging.ERROR if status == TASK_STATUS_FAILED else logging.INFO
|
||||
logger.log(log_level, log_msg)
|
||||
|
||||
# Update task statistics
|
||||
stats_collector.record_task_status_change(old_status, status)
|
||||
def update_task_status(task_id, status, message=None, results=None, error=None, output_dir=None):
|
||||
"""Update task status using the shared implementation (writes to SQLite)."""
|
||||
return shared_update_task_status(task_id, status, message=message, results=results, error=error, output_dir=output_dir)
|
||||
|
||||
|
||||
# Statistics and Monitoring
|
||||
@ -383,180 +373,215 @@ stats_collector = APIStatsCollector()
|
||||
|
||||
def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
"""Background task to process data asynchronously."""
|
||||
logger.info(f"Job {task_id}: Background processing started for task {task_id}")
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Starting data processing...")
|
||||
|
||||
# 1. Load and override config FIRST
|
||||
logger.info(f"Job {task_id}: Loading configuration from {config_path}")
|
||||
config_start = time.time()
|
||||
# 确保后台线程里有 Flask 应用上下文
|
||||
with app.app_context():
|
||||
logger.info(f"Job {task_id}: Background processing started for task {task_id}")
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.safe_load(f)
|
||||
logger.info(f"Job {task_id}: Configuration loaded successfully with {len(config)} keys")
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to load config from {config_path}: {str(e)}")
|
||||
raise
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "开始处理数据...")
|
||||
|
||||
# Update directories based on config output_dir
|
||||
Config.update_directories_from_config(config_path)
|
||||
# 1. Load and override config FIRST
|
||||
logger.info(f"Job {task_id}: Loading configuration from {config_path}")
|
||||
config_start = time.time()
|
||||
|
||||
# Sync app.config with updated directories
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
try:
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.safe_load(f)
|
||||
logger.info(f"Job {task_id}: Configuration loaded successfully with {len(config)} keys")
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to load config from {config_path}: {str(e)}")
|
||||
raise
|
||||
|
||||
# Update task status file path to new output directory
|
||||
from .shared import set_task_status_file_path, load_task_status_from_file
|
||||
set_task_status_file_path(Config.OUTPUT_FOLDER / "task_status.json")
|
||||
# 立即从新路径加载现有状态，避免后续保存清空文件
|
||||
load_task_status_from_file()
|
||||
# Update directories based on config output_dir
|
||||
Config.update_directories_from_config(config_path)
|
||||
|
||||
# Update job directories to be under the correct config-based paths
|
||||
from pathlib import Path
|
||||
job_upload_dir = Path(Config.UPLOAD_FOLDER) / task_id
|
||||
job_output_dir = Path(Config.OUTPUT_FOLDER) / task_id
|
||||
job_upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
job_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Sync app.config with updated directories
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
|
||||
# Move uploaded files to the correct config-based directories
|
||||
try:
|
||||
import shutil
|
||||
# Task status persistence now uses SQLite only
|
||||
# JSON persistence has been disabled
|
||||
# from .shared import set_task_status_file_path, load_task_status_from_file
|
||||
# set_task_status_file_path(Config.OUTPUT_FOLDER / "task_status.json")
|
||||
# load_task_status_from_file()
|
||||
|
||||
# Move data file to correct uploads directory
|
||||
if data_path.parent != job_upload_dir:
|
||||
new_data_path = job_upload_dir / data_path.name
|
||||
if data_path != new_data_path:
|
||||
shutil.move(str(data_path), str(new_data_path))
|
||||
data_path = new_data_path
|
||||
logger.info(f"Job {task_id}: Moved data file to {data_path}")
|
||||
# Update job directories to be under the correct config-based paths
|
||||
from pathlib import Path
|
||||
job_upload_dir = Path(Config.UPLOAD_FOLDER) / task_id
|
||||
job_output_dir = Path(Config.OUTPUT_FOLDER) / task_id
|
||||
job_upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
job_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Move config file to correct uploads directory (if it's a custom config)
|
||||
if config_path.parent != job_upload_dir and config_path.parent != Config.BASE_DIR:
|
||||
new_config_path = job_upload_dir / config_path.name
|
||||
if config_path != new_config_path:
|
||||
shutil.move(str(config_path), str(new_config_path))
|
||||
config_path = new_config_path
|
||||
logger.info(f"Job {task_id}: Moved config file to {config_path}")
|
||||
# Trigger an update to save output_dir to database
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "目录已就绪", output_dir=str(job_output_dir))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Job {task_id}: Failed to move uploaded files to configured directories: {str(e)}")
|
||||
# Move uploaded files to the correct config-based directories
|
||||
try:
|
||||
import shutil
|
||||
|
||||
logger.debug(f"Job {task_id}: Keeping original output directory: {config.get('output_dir', 'not set')}")
|
||||
logger.debug(f"Job {task_id}: Updated directories - Upload: {Config.UPLOAD_FOLDER}, Output: {Config.OUTPUT_FOLDER}, Job output: {job_output_dir}")
|
||||
# Move data file to correct uploads directory
|
||||
if data_path.parent != job_upload_dir:
|
||||
new_data_path = job_upload_dir / data_path.name
|
||||
if data_path != new_data_path:
|
||||
shutil.move(str(data_path), str(new_data_path))
|
||||
data_path = new_data_path
|
||||
logger.info(f"Job {task_id}: Moved data file to {data_path}")
|
||||
|
||||
config_duration = time.time() - config_start
|
||||
logger.info(f"Job {task_id}: Configuration processing completed in {config_duration:.3f}s")
|
||||
# Move config file to correct uploads directory (if it's a custom config)
|
||||
if config_path.parent != job_upload_dir and config_path.parent != Config.BASE_DIR:
|
||||
new_config_path = job_upload_dir / config_path.name
|
||||
if config_path != new_config_path:
|
||||
shutil.move(str(config_path), str(new_config_path))
|
||||
config_path = new_config_path
|
||||
logger.info(f"Job {task_id}: Moved config file to {config_path}")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Configuration loaded, starting preprocessing...")
|
||||
except Exception as e:
|
||||
logger.warning(f"Job {task_id}: Failed to move uploaded files to configured directories: {str(e)}")
|
||||
|
||||
# 2. Data Preprocessing (files are already in correct directories)
|
||||
logger.info(f"Job {task_id}: Starting preprocessing phase...")
|
||||
preprocess_start = time.time()
|
||||
logger.debug(f"Job {task_id}: Using INI configured output directory: {Config.OUTPUT_FOLDER}")
|
||||
logger.debug(f"Job {task_id}: Updated directories - Upload: {Config.UPLOAD_FOLDER}, Output: {Config.OUTPUT_FOLDER}, Job output: {job_output_dir}")
|
||||
|
||||
processed_csv = data_path.parent / f"{data_path.stem}.processed.csv"
|
||||
logger.debug(f"Job {task_id}: Input file: {data_path}, Output file: {processed_csv}")
|
||||
config_duration = time.time() - config_start
|
||||
logger.info(f"Job {task_id}: Configuration processing completed in {config_duration:.3f}s")
|
||||
|
||||
process_file(str(data_path), str(processed_csv), str(config_path))
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "配置已加载，开始预处理...")
|
||||
|
||||
preprocess_duration = time.time() - preprocess_start
|
||||
logger.info(f"Job {task_id}: Preprocessing completed in {preprocess_duration:.3f}s")
|
||||
# 2. Data Preprocessing (files are already in correct directories)
|
||||
logger.info(f"Job {task_id}: Starting preprocessing phase...")
|
||||
preprocess_start = time.time()
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Preprocessing completed, starting GasFlux analysis...")
|
||||
processed_csv = data_path.parent / f"{data_path.stem}.processed.csv"
|
||||
logger.debug(f"Job {task_id}: Input file: {data_path}, Output file: {processed_csv}")
|
||||
|
||||
# Write modified config to a temp file
|
||||
final_config_path = data_path.parent / "final_config.yaml"
|
||||
try:
|
||||
with open(final_config_path, 'w') as f:
|
||||
yaml.safe_dump(config, f)
|
||||
logger.info(f"Job {task_id}: Final config written to {final_config_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to write final config: {str(e)}")
|
||||
raise
|
||||
process_file(str(data_path), str(processed_csv), str(config_path))
|
||||
|
||||
config_duration = time.time() - config_start
|
||||
logger.info(f"Job {task_id}: Configuration processing completed in {config_duration:.3f}s")
|
||||
preprocess_duration = time.time() - preprocess_start
|
||||
logger.info(f"Job {task_id}: Preprocessing completed in {preprocess_duration:.3f}s")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Configuration loaded, starting GasFlux analysis...")
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "预处理完成，开始GasFlux分析...")
|
||||
|
||||
# 3. GasFlux Processing
|
||||
logger.info(f"Job {task_id}: Starting GasFlux analysis...")
|
||||
analysis_start = time.time()
|
||||
# Write modified config to a temp file
|
||||
final_config_path = data_path.parent / "final_config.yaml"
|
||||
try:
|
||||
with open(final_config_path, 'w') as f:
|
||||
yaml.safe_dump(config, f)
|
||||
logger.info(f"Job {task_id}: Final config written to {final_config_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to write final config: {str(e)}")
|
||||
raise
|
||||
|
||||
process_main(processed_csv, final_config_path, task_id)
|
||||
config_duration = time.time() - config_start
|
||||
logger.info(f"Job {task_id}: Configuration processing completed in {config_duration:.3f}s")
|
||||
|
||||
analysis_duration = time.time() - analysis_start
|
||||
logger.info(f"Job {task_id}: GasFlux analysis completed in {analysis_duration:.3f}s")
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "配置已加载，开始GasFlux分析...")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "GasFlux analysis completed, generating reports...")
|
||||
# 3. GasFlux Processing
|
||||
logger.info(f"Job {task_id}: Starting GasFlux analysis...")
|
||||
analysis_start = time.time()
|
||||
|
||||
# Collect results and generate full URLs
|
||||
logger.info(f"Job {task_id}: Collecting generated files from {job_output_dir}")
|
||||
results_start = time.time()
|
||||
results = []
|
||||
processor = process_main(processed_csv, final_config_path, job_output_dir, task_id) # 获取返回值
|
||||
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel_path = f.relative_to(app.config['OUTPUT_FOLDER']).as_posix()
|
||||
file_size = f.stat().st_size
|
||||
results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"size": file_size
|
||||
analysis_duration = time.time() - analysis_start
|
||||
logger.info(f"Job {task_id}: GasFlux analysis completed in {analysis_duration:.3f}s")
|
||||
|
||||
# 提取krig_params数据（只保存关键数值）
|
||||
krig_params_data = []
|
||||
if hasattr(processor, 'output_vars') and 'krig_parameters' in processor.output_vars:
|
||||
for gas, params in processor.output_vars['krig_parameters'].items():
|
||||
# 只保存数值类型的数据，跳过数组
|
||||
clean_params = {}
|
||||
for key, value in params.items():
|
||||
if isinstance(value, (int, float)):
|
||||
clean_params[key] = value
|
||||
elif hasattr(value, 'item') and hasattr(value, 'size'):
|
||||
# numpy数组：只处理单元素数组
|
||||
if value.size == 1:
|
||||
clean_params[key] = value.item()
|
||||
# 多元素数组跳过，不保存
|
||||
elif hasattr(value, 'item'):
|
||||
# 其他numpy对象尝试转换
|
||||
try:
|
||||
clean_params[key] = value.item()
|
||||
except ValueError:
|
||||
# 转换失败则跳过
|
||||
continue
|
||||
|
||||
krig_params_data.append({
|
||||
'gas': gas,
|
||||
'krig_params': clean_params
|
||||
})
|
||||
logger.debug(f"Job {task_id}: Found output file: {f.name} ({file_size} bytes)")
|
||||
|
||||
results_duration = time.time() - results_start
|
||||
logger.info(f"Job {task_id}: Results collection completed in {results_duration:.3f}s - {len(results)} files generated")
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "GasFlux分析完成，正在生成报告...")
|
||||
|
||||
total_size = sum(r.get('size', 0) for r in results)
|
||||
logger.info(f"Job {task_id}: Total output size: {total_size} bytes across {len(results)} files")
|
||||
# Collect results and generate full URLs
|
||||
logger.info(f"Job {task_id}: Collecting generated files from {job_output_dir}")
|
||||
results_start = time.time()
|
||||
results = []
|
||||
|
||||
# 先添加krig_params数据
|
||||
results.extend(krig_params_data)
|
||||
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel_path = f.relative_to(app.config['OUTPUT_FOLDER']).as_posix()
|
||||
file_size = f.stat().st_size
|
||||
results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"size": file_size
|
||||
})
|
||||
logger.debug(f"Job {task_id}: Found output file: {f.name} ({file_size} bytes)")
|
||||
|
||||
results_duration = time.time() - results_start
|
||||
logger.info(f"Job {task_id}: Results collection completed in {results_duration:.3f}s - {len(results)} files generated")
|
||||
|
||||
total_size = sum(r.get('size', 0) for r in results)
|
||||
logger.info(f"Job {task_id}: Total output size: {total_size} bytes across {len(results)} files")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to collect results: {str(e)}")
|
||||
raise
|
||||
|
||||
total_duration = time.time() - start_time
|
||||
logger.info(f"Job {task_id}: Processing complete. Total duration: {total_duration:.3f}s, {len(results)} files generated.")
|
||||
|
||||
# Record task completion time for statistics
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_COMPLETED, "处理成功完成", results=results)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to collect results: {str(e)}")
|
||||
raise
|
||||
total_duration = time.time() - start_time
|
||||
logger.error(f"Job {task_id}: Processing failed after {total_duration:.3f}s - Error: {str(e)}", exc_info=True)
|
||||
|
||||
total_duration = time.time() - start_time
|
||||
logger.info(f"Job {task_id}: Processing complete. Total duration: {total_duration:.3f}s, {len(results)} files generated.")
|
||||
# Record failed task processing time for statistics
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
logger.error(f"Job {task_id}: Failed task details - Data: {data_path}, Config: {config_path}, Output: {job_output_dir}")
|
||||
|
||||
# Record task completion time for statistics
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
# Try to capture any partial results
|
||||
partial_results = []
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel_path = f.relative_to(app.config['OUTPUT_FOLDER']).as_posix()
|
||||
partial_results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"size": f.stat().st_size,
|
||||
"note": "partial_result"
|
||||
})
|
||||
except Exception as collect_error:
|
||||
logger.warning(f"Job {task_id}: Failed to collect partial results: {str(collect_error)}")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_COMPLETED, "Processing completed successfully", results=results)
|
||||
error_msg = f"处理失败: {str(e)}"
|
||||
if partial_results:
|
||||
error_msg += f" (部分结果可用: {len(partial_results)} 个文件)"
|
||||
|
||||
except Exception as e:
|
||||
total_duration = time.time() - start_time
|
||||
logger.error(f"Job {task_id}: Processing failed after {total_duration:.3f}s - Error: {str(e)}", exc_info=True)
|
||||
|
||||
# Record failed task processing time for statistics
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
logger.error(f"Job {task_id}: Failed task details - Data: {data_path}, Config: {config_path}, Output: {job_output_dir}")
|
||||
|
||||
# Try to capture any partial results
|
||||
partial_results = []
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel_path = f.relative_to(app.config['OUTPUT_FOLDER']).as_posix()
|
||||
partial_results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"size": f.stat().st_size,
|
||||
"note": "partial_result"
|
||||
})
|
||||
except Exception as collect_error:
|
||||
logger.warning(f"Job {task_id}: Failed to collect partial results: {str(collect_error)}")
|
||||
|
||||
error_msg = f"Processing failed: {str(e)}"
|
||||
if partial_results:
|
||||
error_msg += f" (partial results available: {len(partial_results)} files)"
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_FAILED, error=error_msg, results=partial_results if partial_results else None)
|
||||
update_task_status(task_id, TASK_STATUS_FAILED, error=error_msg, results=partial_results if partial_results else None)
|
||||
|
||||
# Import GasFlux modules
|
||||
logger.info("Importing GasFlux modules...")
|
||||
@ -632,10 +657,20 @@ Config.init_base_dir()
|
||||
# ALLOWED_DATA_EXTENSIONS and ALLOWED_CONFIG_EXTENSIONS moved to shared.py
|
||||
|
||||
app.config['MAX_CONTENT_LENGTH'] = Config.MAX_CONTENT_LENGTH
|
||||
# Don't set UPLOAD_FOLDER and OUTPUT_FOLDER here - they will be set dynamically per request
|
||||
# Set defaults to avoid KeyError if any handler reads before config is applied
|
||||
app.config.setdefault('UPLOAD_FOLDER', None)
|
||||
app.config.setdefault('OUTPUT_FOLDER', None)
|
||||
app.config['BASE_URL'] = Config.BASE_URL
|
||||
|
||||
# Set upload and output folders from config
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
|
||||
# Database and persistence configuration
|
||||
if Config.DB_PATH:
|
||||
app.config['DB_PATH'] = Config.DB_PATH
|
||||
if Config.TASK_PERSIST_BACKEND:
|
||||
app.config['TASK_PERSIST_BACKEND'] = Config.TASK_PERSIST_BACKEND
|
||||
app.config['JANITOR_DRY_RUN'] = str(Config.JANITOR_DRY_RUN).lower()
|
||||
if Config.ADMIN_BOOTSTRAP_KEY:
|
||||
app.config['ADMIN_BOOTSTRAP_KEY'] = Config.ADMIN_BOOTSTRAP_KEY
|
||||
|
||||
# Log current configuration
|
||||
logger.info(f"Upload folder: {Config.UPLOAD_FOLDER}")
|
||||
@ -673,6 +708,10 @@ def setup_directories():
|
||||
|
||||
# allowed_file moved to shared.py
|
||||
|
||||
# Initialize database and start background services
|
||||
from .db import init_app as init_db
|
||||
init_db(app)
|
||||
|
||||
# Import blueprints after app initialization to avoid circular imports
|
||||
from .blueprints.health import health_bp
|
||||
from .blueprints.upload import upload_bp
|
||||
@ -683,6 +722,7 @@ from .blueprints.config import config_bp
|
||||
from .blueprints.reports import reports_bp
|
||||
from .blueprints.download import download_bp
|
||||
from .blueprints.web import web_bp
|
||||
from .blueprints.api_keys import api_keys_bp
|
||||
|
||||
# Register blueprints
|
||||
app.register_blueprint(health_bp)
|
||||
@ -694,30 +734,18 @@ app.register_blueprint(config_bp)
|
||||
app.register_blueprint(reports_bp)
|
||||
app.register_blueprint(download_bp)
|
||||
app.register_blueprint(web_bp)
|
||||
app.register_blueprint(api_keys_bp)
|
||||
|
||||
# Load persisted task status after app initialization
|
||||
# Task status persistence now uses SQLite only
|
||||
# JSON persistence has been disabled - functions removed from shared.py
|
||||
|
||||
# Initialize janitor for background cleanup
|
||||
try:
|
||||
from .shared import (
|
||||
load_task_status_from_file,
|
||||
save_task_status_to_file,
|
||||
set_task_status_file_path,
|
||||
)
|
||||
|
||||
# 只有在 OUTPUT_FOLDER 有效时才启用持久化
|
||||
if hasattr(Config, 'OUTPUT_FOLDER') and Config.OUTPUT_FOLDER:
|
||||
task_status_path = Config.OUTPUT_FOLDER / "task_status.json"
|
||||
set_task_status_file_path(task_status_path)
|
||||
logger.info(f"Task status persistence path set to: {task_status_path}")
|
||||
with app.app_context():
|
||||
load_task_status_from_file()
|
||||
|
||||
import atexit
|
||||
def _save_on_exit():
|
||||
with app.app_context():
|
||||
save_task_status_to_file()
|
||||
atexit.register(_save_on_exit)
|
||||
else:
|
||||
logger.info("Task status persistence will be configured after config is loaded")
|
||||
from .janitor import start_janitor, reconcile_tasks_on_startup
|
||||
with app.app_context():
|
||||
reconcile_tasks_on_startup()
|
||||
# No longer need to load task status into memory
|
||||
start_janitor(app)
|
||||
except Exception as e:
|
||||
print(f"⚠ Failed to setup task persistence: {e}")
|
||||
|
||||
@ -725,4 +753,4 @@ except Exception as e:
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=5000, debug=False)
|
||||
app.run(host=Config.HOST, port=Config.PORT, debug=Config.DEBUG)
|
||||
|
||||
Reference in New Issue
Block a user