增加web_api
This commit is contained in:
728
src/gasflux/app.py
Normal file
728
src/gasflux/app.py
Normal file
@ -0,0 +1,728 @@
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import uuid
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from flask import Flask, request, jsonify, send_file, render_template_string, url_for, g
|
||||
from flask_cors import CORS
|
||||
from werkzeug.utils import secure_filename
|
||||
import yaml
|
||||
|
||||
# Shared utilities imported from shared.py
|
||||
try:
|
||||
# Try relative import (when run as part of package)
|
||||
from .shared import task_status, TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED
|
||||
except ImportError:
|
||||
# Fallback to absolute import (when run directly)
|
||||
from shared import task_status, TASK_STATUS_PENDING, TASK_STATUS_PROCESSING, TASK_STATUS_COMPLETED, TASK_STATUS_FAILED
|
||||
|
||||
# Blueprints will be imported after app initialization to avoid circular imports
|
||||
|
||||
# Environment-based configuration management
|
||||
class Config:
|
||||
"""Configuration management using environment variables with defaults."""
|
||||
|
||||
# Server configuration
|
||||
HOST = os.getenv('GASFLUX_HOST', '0.0.0.0')
|
||||
PORT = int(os.getenv('GASFLUX_PORT', '5000'))
|
||||
DEBUG = os.getenv('GASFLUX_DEBUG', 'false').lower() in ('true', '1', 'yes', 'on')
|
||||
|
||||
# Directory configuration
|
||||
BASE_DIR = None # Will be set dynamically
|
||||
UPLOAD_FOLDER_NAME = os.getenv('GASFLUX_UPLOAD_FOLDER', 'web_api_data/uploads')
|
||||
OUTPUT_FOLDER_NAME = os.getenv('GASFLUX_OUTPUT_FOLDER', 'web_api_data/outputs')
|
||||
|
||||
# File size limits (in bytes)
|
||||
MAX_CONTENT_LENGTH = int(os.getenv('GASFLUX_MAX_CONTENT_LENGTH', str(100 * 1024 * 1024))) # 100MB
|
||||
|
||||
# Logging configuration
|
||||
LOG_LEVEL = os.getenv('GASFLUX_LOG_LEVEL', 'INFO').upper()
|
||||
LOG_FILE = os.getenv('GASFLUX_LOG_FILE', 'logs/gasflux_api.log')
|
||||
|
||||
# CORS configuration
|
||||
CORS_ORIGINS = os.getenv('GASFLUX_CORS_ORIGINS', '*').split(',')
|
||||
|
||||
# Task management
|
||||
TASK_CLEANUP_INTERVAL = int(os.getenv('GASFLUX_TASK_CLEANUP_INTERVAL', '3600')) # 1 hour in seconds
|
||||
MAX_TASK_AGE = int(os.getenv('GASFLUX_MAX_TASK_AGE', str(24 * 3600))) # 24 hours in seconds
|
||||
|
||||
# Performance tuning
|
||||
THREADS = int(os.getenv('GASFLUX_THREADS', '8')) # Waitress threads
|
||||
CONNECTION_LIMIT = int(os.getenv('GASFLUX_CONNECTION_LIMIT', '100'))
|
||||
CHANNEL_TIMEOUT = int(os.getenv('GASFLUX_CHANNEL_TIMEOUT', '300')) # 5 minutes
|
||||
|
||||
@classmethod
|
||||
def init_base_dir(cls):
|
||||
"""Initialize base directory based on environment."""
|
||||
try:
|
||||
if getattr(sys, 'frozen', False):
|
||||
# Running in PyInstaller bundle
|
||||
cls.BASE_DIR = Path(sys.executable).parent
|
||||
else:
|
||||
# Running in normal Python environment
|
||||
cls.BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
except:
|
||||
# Fallback to current working directory
|
||||
cls.BASE_DIR = Path.cwd()
|
||||
|
||||
# Initialize directories based on config
|
||||
cls.init_directories()
|
||||
|
||||
@classmethod
|
||||
def init_directories(cls, output_dir=None):
|
||||
"""Initialize upload and output directories."""
|
||||
if output_dir:
|
||||
# Use config-based output directory
|
||||
output_base = Path(output_dir)
|
||||
if not output_base.is_absolute():
|
||||
output_base = cls.BASE_DIR / output_base
|
||||
else:
|
||||
# Use default relative paths
|
||||
output_base = cls.BASE_DIR
|
||||
|
||||
# Set upload and output directories relative to output_base
|
||||
cls.UPLOAD_FOLDER = output_base / "uploads"
|
||||
cls.OUTPUT_FOLDER = output_base / "outputs"
|
||||
|
||||
# Create directories
|
||||
cls.UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
cls.OUTPUT_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Directories initialized - Upload: {cls.UPLOAD_FOLDER}, Output: {cls.OUTPUT_FOLDER}")
|
||||
|
||||
@classmethod
|
||||
def update_directories_from_config(cls, config_path=None):
|
||||
"""Update directories based on config file."""
|
||||
if config_path and Path(config_path).exists():
|
||||
try:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
config = yaml.safe_load(f)
|
||||
output_dir = config.get('output_dir')
|
||||
if output_dir:
|
||||
cls.init_directories(output_dir)
|
||||
logger.info(f"Directories updated from config: {config_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update directories from config {config_path}: {e}")
|
||||
else:
|
||||
logger.info("Using default directory configuration")
|
||||
|
||||
@classmethod
|
||||
def get_log_level(cls):
|
||||
"""Get logging level from string."""
|
||||
levels = {
|
||||
'DEBUG': logging.DEBUG,
|
||||
'INFO': logging.INFO,
|
||||
'WARNING': logging.WARNING,
|
||||
'ERROR': logging.ERROR,
|
||||
'CRITICAL': logging.CRITICAL
|
||||
}
|
||||
return levels.get(cls.LOG_LEVEL, logging.INFO)
|
||||
|
||||
@classmethod
|
||||
def to_dict(cls):
|
||||
"""Return configuration as dictionary for debugging."""
|
||||
return {
|
||||
'host': cls.HOST,
|
||||
'port': cls.PORT,
|
||||
'debug': cls.DEBUG,
|
||||
'base_dir': str(cls.BASE_DIR) if cls.BASE_DIR else None,
|
||||
'upload_folder': str(cls.UPLOAD_FOLDER) if hasattr(cls, 'UPLOAD_FOLDER') else None,
|
||||
'output_folder': str(cls.OUTPUT_FOLDER) if hasattr(cls, 'OUTPUT_FOLDER') else None,
|
||||
'max_content_length': cls.MAX_CONTENT_LENGTH,
|
||||
'log_level': cls.LOG_LEVEL,
|
||||
'log_file': cls.LOG_FILE,
|
||||
'cors_origins': cls.CORS_ORIGINS,
|
||||
'task_cleanup_interval': cls.TASK_CLEANUP_INTERVAL,
|
||||
'max_task_age': cls.MAX_TASK_AGE,
|
||||
'threads': cls.THREADS,
|
||||
'connection_limit': cls.CONNECTION_LIMIT,
|
||||
'channel_timeout': cls.CHANNEL_TIMEOUT
|
||||
}
|
||||
|
||||
# Initialize logging with environment-based configuration
|
||||
logging.basicConfig(
|
||||
level=Config.get_log_level(),
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(), # Console output
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger("gasflux_api")
|
||||
logger.info("Basic logging initialized")
|
||||
|
||||
|
||||
def log_performance(func):
|
||||
"""Decorator to log function performance."""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
start_time = time.time()
|
||||
func_name = func.__name__
|
||||
logger.debug(f"PERF: Starting {func_name}")
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
duration = time.time() - start_time
|
||||
logger.info(f"PERF: {func_name} completed in {duration:.3f}s")
|
||||
return result
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"PERF: {func_name} failed after {duration:.3f}s - Error: {str(e)}")
|
||||
raise
|
||||
return wrapper
|
||||
|
||||
# Task status management
|
||||
# Task status constants and storage moved to shared.py
|
||||
|
||||
def update_task_status(task_id, status, message=None, results=None, error=None):
|
||||
"""Update task status in the global dictionary."""
|
||||
timestamp = time.time()
|
||||
old_status = task_status.get(task_id, {}).get("status", "unknown")
|
||||
|
||||
task_status[task_id] = {
|
||||
"status": status,
|
||||
"message": message,
|
||||
"results": results,
|
||||
"error": error,
|
||||
"updated_at": timestamp
|
||||
}
|
||||
|
||||
# Log detailed status change with context
|
||||
log_msg = f"Task {task_id} status changed: {old_status} -> {status}"
|
||||
if message:
|
||||
log_msg += f" | Message: {message}"
|
||||
if results:
|
||||
log_msg += f" | Results count: {len(results) if isinstance(results, list) else 'N/A'}"
|
||||
if error:
|
||||
log_msg += f" | Error: {error}"
|
||||
|
||||
log_level = logging.ERROR if status == TASK_STATUS_FAILED else logging.INFO
|
||||
logger.log(log_level, log_msg)
|
||||
|
||||
# Update task statistics
|
||||
stats_collector.record_task_status_change(old_status, status)
|
||||
|
||||
|
||||
# Statistics and Monitoring
|
||||
class APIStatsCollector:
|
||||
"""Collect and manage API statistics."""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = time.time()
|
||||
self.reset_stats()
|
||||
|
||||
def reset_stats(self):
|
||||
"""Reset all statistics."""
|
||||
self.stats = {
|
||||
'requests': {
|
||||
'total': 0,
|
||||
'by_method': {},
|
||||
'by_endpoint': {},
|
||||
'by_status': {},
|
||||
'response_times': [],
|
||||
'errors': 0
|
||||
},
|
||||
'tasks': {
|
||||
'total_created': 0,
|
||||
'total_completed': 0,
|
||||
'total_failed': 0,
|
||||
'by_status': {
|
||||
'pending': 0,
|
||||
'processing': 0,
|
||||
'completed': 0,
|
||||
'failed': 0
|
||||
},
|
||||
'processing_times': []
|
||||
},
|
||||
'performance': {
|
||||
'avg_response_time': 0,
|
||||
'max_response_time': 0,
|
||||
'min_response_time': float('inf'),
|
||||
'uptime_seconds': time.time() - self.start_time
|
||||
}
|
||||
}
|
||||
|
||||
def record_request(self, method, endpoint, status_code, response_time):
|
||||
"""Record an API request."""
|
||||
self.stats['requests']['total'] += 1
|
||||
|
||||
# Method stats
|
||||
if method not in self.stats['requests']['by_method']:
|
||||
self.stats['requests']['by_method'][method] = 0
|
||||
self.stats['requests']['by_method'][method] += 1
|
||||
|
||||
# Endpoint stats
|
||||
if endpoint not in self.stats['requests']['by_endpoint']:
|
||||
self.stats['requests']['by_endpoint'][endpoint] = 0
|
||||
self.stats['requests']['by_endpoint'][endpoint] += 1
|
||||
|
||||
# Status stats
|
||||
status_category = str(status_code // 100 * 100) # 200, 400, 500, etc.
|
||||
if status_category not in self.stats['requests']['by_status']:
|
||||
self.stats['requests']['by_status'][status_category] = 0
|
||||
self.stats['requests']['by_status'][status_category] += 1
|
||||
|
||||
# Response time stats
|
||||
self.stats['requests']['response_times'].append(response_time)
|
||||
|
||||
# Keep only last 1000 response times for memory efficiency
|
||||
if len(self.stats['requests']['response_times']) > 1000:
|
||||
self.stats['requests']['response_times'] = self.stats['requests']['response_times'][-1000:]
|
||||
|
||||
# Error tracking
|
||||
if status_code >= 400:
|
||||
self.stats['requests']['errors'] += 1
|
||||
|
||||
# Update performance stats
|
||||
self._update_performance_stats()
|
||||
|
||||
def record_task_status_change(self, old_status, new_status):
|
||||
"""Record task status changes."""
|
||||
if old_status == "unknown": # New task
|
||||
self.stats['tasks']['total_created'] += 1
|
||||
|
||||
if new_status == TASK_STATUS_COMPLETED:
|
||||
self.stats['tasks']['total_completed'] += 1
|
||||
elif new_status == TASK_STATUS_FAILED:
|
||||
self.stats['tasks']['total_failed'] += 1
|
||||
|
||||
# Update status counts
|
||||
for status in [old_status, new_status]:
|
||||
if status in self.stats['tasks']['by_status']:
|
||||
if status == old_status and old_status != "unknown":
|
||||
self.stats['tasks']['by_status'][old_status] -= 1
|
||||
elif status == new_status:
|
||||
self.stats['tasks']['by_status'][new_status] += 1
|
||||
|
||||
def record_task_completion_time(self, completion_time):
|
||||
"""Record task completion time."""
|
||||
self.stats['tasks']['processing_times'].append(completion_time)
|
||||
|
||||
# Keep only last 100 processing times
|
||||
if len(self.stats['tasks']['processing_times']) > 100:
|
||||
self.stats['tasks']['processing_times'] = self.stats['tasks']['processing_times'][-100:]
|
||||
|
||||
def _update_performance_stats(self):
|
||||
"""Update performance statistics."""
|
||||
response_times = self.stats['requests']['response_times']
|
||||
if response_times:
|
||||
self.stats['performance']['avg_response_time'] = sum(response_times) / len(response_times)
|
||||
self.stats['performance']['max_response_time'] = max(response_times)
|
||||
self.stats['performance']['min_response_time'] = min(response_times)
|
||||
|
||||
self.stats['performance']['uptime_seconds'] = time.time() - self.start_time
|
||||
|
||||
def get_summary(self):
|
||||
"""Get a summary of current statistics."""
|
||||
current_time = time.time()
|
||||
uptime = current_time - self.start_time
|
||||
|
||||
# Calculate rates
|
||||
requests_per_second = self.stats['requests']['total'] / max(uptime, 1)
|
||||
error_rate = (self.stats['requests']['errors'] / max(self.stats['requests']['total'], 1)) * 100
|
||||
|
||||
# Task completion rate
|
||||
total_tasks_processed = self.stats['tasks']['total_completed'] + self.stats['tasks']['total_failed']
|
||||
task_success_rate = (self.stats['tasks']['total_completed'] / max(total_tasks_processed, 1)) * 100
|
||||
|
||||
return {
|
||||
'summary': {
|
||||
'uptime_seconds': uptime,
|
||||
'uptime_formatted': self._format_uptime(uptime),
|
||||
'requests_total': self.stats['requests']['total'],
|
||||
'requests_per_second': round(requests_per_second, 2),
|
||||
'error_rate_percent': round(error_rate, 2),
|
||||
'active_tasks': len([t for t in task_status.values()
|
||||
if t.get('status') in [TASK_STATUS_PENDING, TASK_STATUS_PROCESSING]])
|
||||
},
|
||||
'requests': {
|
||||
'by_method': self.stats['requests']['by_method'],
|
||||
'by_status': self.stats['requests']['by_status'],
|
||||
'top_endpoints': dict(sorted(self.stats['requests']['by_endpoint'].items(),
|
||||
key=lambda x: x[1], reverse=True)[:10])
|
||||
},
|
||||
'tasks': {
|
||||
'total_created': self.stats['tasks']['total_created'],
|
||||
'total_completed': self.stats['tasks']['total_completed'],
|
||||
'total_failed': self.stats['tasks']['total_failed'],
|
||||
'success_rate_percent': round(task_success_rate, 2),
|
||||
'by_status': self.stats['tasks']['by_status']
|
||||
},
|
||||
'performance': {
|
||||
'avg_response_time_ms': round(self.stats['performance']['avg_response_time'] * 1000, 2),
|
||||
'max_response_time_ms': round(self.stats['performance']['max_response_time'] * 1000, 2),
|
||||
'min_response_time_ms': round(self.stats['performance']['min_response_time'] * 1000, 2) if self.stats['performance']['min_response_time'] != float('inf') else 0
|
||||
}
|
||||
}
|
||||
|
||||
def _format_uptime(self, seconds):
|
||||
"""Format uptime in human readable format."""
|
||||
days, remainder = divmod(int(seconds), 86400)
|
||||
hours, remainder = divmod(remainder, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
|
||||
parts = []
|
||||
if days > 0:
|
||||
parts.append(f"{days}d")
|
||||
if hours > 0:
|
||||
parts.append(f"{hours}h")
|
||||
if minutes > 0:
|
||||
parts.append(f"{minutes}m")
|
||||
parts.append(f"{seconds}s")
|
||||
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
# Global statistics collector
|
||||
stats_collector = APIStatsCollector()
|
||||
|
||||
# get_task_status moved to shared.py
|
||||
|
||||
# cleanup_old_tasks moved to shared.py
|
||||
|
||||
def process_data_async(task_id, data_path, config_path, job_output_dir):
|
||||
"""Background task to process data asynchronously."""
|
||||
logger.info(f"Job {task_id}: Background processing started for task {task_id}")
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Starting data processing...")
|
||||
|
||||
# 1. Load and override config FIRST
|
||||
logger.info(f"Job {task_id}: Loading configuration from {config_path}")
|
||||
config_start = time.time()
|
||||
|
||||
try:
|
||||
with open(config_path, 'r') as f:
|
||||
config = yaml.safe_load(f)
|
||||
logger.info(f"Job {task_id}: Configuration loaded successfully with {len(config)} keys")
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to load config from {config_path}: {str(e)}")
|
||||
raise
|
||||
|
||||
# Update directories based on config output_dir
|
||||
Config.update_directories_from_config(config_path)
|
||||
|
||||
# Sync app.config with updated directories
|
||||
app.config['UPLOAD_FOLDER'] = Config.UPLOAD_FOLDER
|
||||
app.config['OUTPUT_FOLDER'] = Config.OUTPUT_FOLDER
|
||||
|
||||
# Update task status file path to new output directory
|
||||
from .shared import set_task_status_file_path, load_task_status_from_file
|
||||
set_task_status_file_path(Config.OUTPUT_FOLDER / "task_status.json")
|
||||
# 立即从新路径加载现有状态，避免后续保存清空文件
|
||||
load_task_status_from_file()
|
||||
|
||||
# Update job directories to be under the correct config-based paths
|
||||
from pathlib import Path
|
||||
job_upload_dir = Path(Config.UPLOAD_FOLDER) / task_id
|
||||
job_output_dir = Path(Config.OUTPUT_FOLDER) / task_id
|
||||
job_upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
job_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Move uploaded files to the correct config-based directories
|
||||
try:
|
||||
import shutil
|
||||
|
||||
# Move data file to correct uploads directory
|
||||
if data_path.parent != job_upload_dir:
|
||||
new_data_path = job_upload_dir / data_path.name
|
||||
if data_path != new_data_path:
|
||||
shutil.move(str(data_path), str(new_data_path))
|
||||
data_path = new_data_path
|
||||
logger.info(f"Job {task_id}: Moved data file to {data_path}")
|
||||
|
||||
# Move config file to correct uploads directory (if it's a custom config)
|
||||
if config_path.parent != job_upload_dir and config_path.parent != Config.BASE_DIR:
|
||||
new_config_path = job_upload_dir / config_path.name
|
||||
if config_path != new_config_path:
|
||||
shutil.move(str(config_path), str(new_config_path))
|
||||
config_path = new_config_path
|
||||
logger.info(f"Job {task_id}: Moved config file to {config_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Job {task_id}: Failed to move uploaded files to configured directories: {str(e)}")
|
||||
|
||||
logger.debug(f"Job {task_id}: Keeping original output directory: {config.get('output_dir', 'not set')}")
|
||||
logger.debug(f"Job {task_id}: Updated directories - Upload: {Config.UPLOAD_FOLDER}, Output: {Config.OUTPUT_FOLDER}, Job output: {job_output_dir}")
|
||||
|
||||
config_duration = time.time() - config_start
|
||||
logger.info(f"Job {task_id}: Configuration processing completed in {config_duration:.3f}s")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Configuration loaded, starting preprocessing...")
|
||||
|
||||
# 2. Data Preprocessing (files are already in correct directories)
|
||||
logger.info(f"Job {task_id}: Starting preprocessing phase...")
|
||||
preprocess_start = time.time()
|
||||
|
||||
processed_csv = data_path.parent / f"{data_path.stem}.processed.csv"
|
||||
logger.debug(f"Job {task_id}: Input file: {data_path}, Output file: {processed_csv}")
|
||||
|
||||
process_file(str(data_path), str(processed_csv), str(config_path))
|
||||
|
||||
preprocess_duration = time.time() - preprocess_start
|
||||
logger.info(f"Job {task_id}: Preprocessing completed in {preprocess_duration:.3f}s")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Preprocessing completed, starting GasFlux analysis...")
|
||||
|
||||
# Write modified config to a temp file
|
||||
final_config_path = data_path.parent / "final_config.yaml"
|
||||
try:
|
||||
with open(final_config_path, 'w') as f:
|
||||
yaml.safe_dump(config, f)
|
||||
logger.info(f"Job {task_id}: Final config written to {final_config_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to write final config: {str(e)}")
|
||||
raise
|
||||
|
||||
config_duration = time.time() - config_start
|
||||
logger.info(f"Job {task_id}: Configuration processing completed in {config_duration:.3f}s")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "Configuration loaded, starting GasFlux analysis...")
|
||||
|
||||
# 3. GasFlux Processing
|
||||
logger.info(f"Job {task_id}: Starting GasFlux analysis...")
|
||||
analysis_start = time.time()
|
||||
|
||||
process_main(processed_csv, final_config_path, task_id)
|
||||
|
||||
analysis_duration = time.time() - analysis_start
|
||||
logger.info(f"Job {task_id}: GasFlux analysis completed in {analysis_duration:.3f}s")
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_PROCESSING, "GasFlux analysis completed, generating reports...")
|
||||
|
||||
# Collect results and generate full URLs
|
||||
logger.info(f"Job {task_id}: Collecting generated files from {job_output_dir}")
|
||||
results_start = time.time()
|
||||
results = []
|
||||
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel_path = f.relative_to(app.config['OUTPUT_FOLDER']).as_posix()
|
||||
file_size = f.stat().st_size
|
||||
results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"size": file_size
|
||||
})
|
||||
logger.debug(f"Job {task_id}: Found output file: {f.name} ({file_size} bytes)")
|
||||
|
||||
results_duration = time.time() - results_start
|
||||
logger.info(f"Job {task_id}: Results collection completed in {results_duration:.3f}s - {len(results)} files generated")
|
||||
|
||||
total_size = sum(r.get('size', 0) for r in results)
|
||||
logger.info(f"Job {task_id}: Total output size: {total_size} bytes across {len(results)} files")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {task_id}: Failed to collect results: {str(e)}")
|
||||
raise
|
||||
|
||||
total_duration = time.time() - start_time
|
||||
logger.info(f"Job {task_id}: Processing complete. Total duration: {total_duration:.3f}s, {len(results)} files generated.")
|
||||
|
||||
# Record task completion time for statistics
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_COMPLETED, "Processing completed successfully", results=results)
|
||||
|
||||
except Exception as e:
|
||||
total_duration = time.time() - start_time
|
||||
logger.error(f"Job {task_id}: Processing failed after {total_duration:.3f}s - Error: {str(e)}", exc_info=True)
|
||||
|
||||
# Record failed task processing time for statistics
|
||||
stats_collector.record_task_completion_time(total_duration)
|
||||
logger.error(f"Job {task_id}: Failed task details - Data: {data_path}, Config: {config_path}, Output: {job_output_dir}")
|
||||
|
||||
# Try to capture any partial results
|
||||
partial_results = []
|
||||
try:
|
||||
for f in job_output_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel_path = f.relative_to(app.config['OUTPUT_FOLDER']).as_posix()
|
||||
partial_results.append({
|
||||
"name": f.name,
|
||||
"rel_path": rel_path,
|
||||
"download_url": f"/download/{rel_path}", # Relative URL that client can use
|
||||
"size": f.stat().st_size,
|
||||
"note": "partial_result"
|
||||
})
|
||||
except Exception as collect_error:
|
||||
logger.warning(f"Job {task_id}: Failed to collect partial results: {str(collect_error)}")
|
||||
|
||||
error_msg = f"Processing failed: {str(e)}"
|
||||
if partial_results:
|
||||
error_msg += f" (partial results available: {len(partial_results)} files)"
|
||||
|
||||
update_task_status(task_id, TASK_STATUS_FAILED, error=error_msg, results=partial_results if partial_results else None)
|
||||
|
||||
# Import GasFlux modules
|
||||
logger.info("Importing GasFlux modules...")
|
||||
import_start = time.time()
|
||||
|
||||
try:
|
||||
# Try absolute imports first (more reliable)
|
||||
from src.gasflux.processing_pipelines import process_main
|
||||
from src.gasflux.data_processor import process_file
|
||||
from src.gasflux.reporting import generate_reports
|
||||
import_duration = time.time() - import_start
|
||||
logger.info(f"GasFlux modules imported successfully in {import_duration:.3f}s (absolute import)")
|
||||
except ImportError as e1:
|
||||
logger.warning(f"Absolute import failed, trying relative import: {e1}")
|
||||
try:
|
||||
from .processing_pipelines import process_main
|
||||
from .data_processor import process_file
|
||||
from .reporting import generate_reports
|
||||
import_duration = time.time() - import_start
|
||||
logger.info(f"GasFlux modules imported successfully in {import_duration:.3f}s (relative import)")
|
||||
except ImportError as e2:
|
||||
import_duration = time.time() - import_start
|
||||
logger.error(f"Failed to import GasFlux modules after {import_duration:.3f}s - Absolute error: {e1}, Relative error: {e2}")
|
||||
raise ImportError(f"Cannot import GasFlux modules: {e2}")
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app) # Initialize CORS
|
||||
|
||||
# Enhanced logging configuration after app initialization
|
||||
try:
|
||||
log_file_path = Path(Config.LOG_FILE)
|
||||
log_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create file handler
|
||||
file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
|
||||
file_handler.setLevel(Config.get_log_level())
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
# Add file handler to logger
|
||||
logger.addHandler(file_handler)
|
||||
logger.info(f"File logging initialized. Log file: {log_file_path.absolute()}")
|
||||
print(f"Log file: {log_file_path.absolute()}") # Also print to console
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to initialize file logging: {e}")
|
||||
logger.warning(f"Failed to initialize file logging: {e}")
|
||||
|
||||
logger.info("Flask application initialized")
|
||||
|
||||
# Request logging middleware
|
||||
@app.before_request
|
||||
def log_request_info():
|
||||
"""Log incoming request details."""
|
||||
g.start_time = time.time()
|
||||
logger.info(f"REQUEST: {request.method} {request.url} - IP: {request.remote_addr} - User-Agent: {request.headers.get('User-Agent', 'Unknown')}")
|
||||
|
||||
@app.after_request
|
||||
def log_response_info(response):
|
||||
"""Log response details."""
|
||||
duration = time.time() - g.start_time
|
||||
logger.info(f"RESPONSE: {request.method} {request.url} - Status: {response.status_code} - Duration: {duration:.3f}s")
|
||||
|
||||
# Record statistics
|
||||
endpoint = request.url_rule.rule if request.url_rule else request.path
|
||||
stats_collector.record_request(request.method, endpoint, response.status_code, duration)
|
||||
|
||||
return response
|
||||
|
||||
# Initialize configuration from environment variables
|
||||
Config.init_base_dir()
|
||||
|
||||
# Apply configuration to app (directories will be created dynamically based on config)
|
||||
# ALLOWED_DATA_EXTENSIONS and ALLOWED_CONFIG_EXTENSIONS moved to shared.py
|
||||
|
||||
app.config['MAX_CONTENT_LENGTH'] = Config.MAX_CONTENT_LENGTH
|
||||
# Don't set UPLOAD_FOLDER and OUTPUT_FOLDER here - they will be set dynamically per request
|
||||
# Set defaults to avoid KeyError if any handler reads before config is applied
|
||||
app.config.setdefault('UPLOAD_FOLDER', None)
|
||||
app.config.setdefault('OUTPUT_FOLDER', None)
|
||||
|
||||
# Log current configuration
|
||||
logger.info(f"Upload folder: {Config.UPLOAD_FOLDER}")
|
||||
logger.info(f"Output folder: {Config.OUTPUT_FOLDER}")
|
||||
logger.info(f"Configuration: {Config.to_dict()}")
|
||||
|
||||
# Ensure directories exist at startup
|
||||
def setup_directories():
|
||||
logger.info("Initializing application directories...")
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Check if directories already exist
|
||||
upload_exists = Config.UPLOAD_FOLDER.exists()
|
||||
output_exists = Config.OUTPUT_FOLDER.exists()
|
||||
|
||||
Config.UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
Config.OUTPUT_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
duration = time.time() - start_time
|
||||
logger.info(f"Directories initialized in {duration:.3f}s: {Config.UPLOAD_FOLDER} ({'existing' if upload_exists else 'created'}), {Config.OUTPUT_FOLDER} ({'existing' if output_exists else 'created'})")
|
||||
|
||||
# Log directory permissions
|
||||
upload_writable = os.access(Config.UPLOAD_FOLDER, os.W_OK)
|
||||
output_writable = os.access(Config.OUTPUT_FOLDER, os.W_OK)
|
||||
logger.info(f"Directory permissions - Upload writable: {upload_writable}, Output writable: {output_writable}")
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"Failed to create directories after {duration:.3f}s: {e}")
|
||||
raise
|
||||
|
||||
# setup_directories() - commented out to avoid creating directories at startup
|
||||
# Directories will be created dynamically based on config when processing tasks
|
||||
|
||||
# allowed_file moved to shared.py
|
||||
|
||||
# Import blueprints after app initialization to avoid circular imports
|
||||
from .blueprints.health import health_bp
|
||||
from .blueprints.upload import upload_bp
|
||||
from .blueprints.tasks import tasks_bp
|
||||
from .blueprints.task_pool import task_pool_bp
|
||||
from .blueprints.stats import stats_bp
|
||||
from .blueprints.config import config_bp
|
||||
from .blueprints.reports import reports_bp
|
||||
from .blueprints.download import download_bp
|
||||
from .blueprints.web import web_bp
|
||||
|
||||
# Register blueprints
|
||||
app.register_blueprint(health_bp)
|
||||
app.register_blueprint(upload_bp)
|
||||
app.register_blueprint(tasks_bp)
|
||||
app.register_blueprint(task_pool_bp)
|
||||
app.register_blueprint(stats_bp)
|
||||
app.register_blueprint(config_bp)
|
||||
app.register_blueprint(reports_bp)
|
||||
app.register_blueprint(download_bp)
|
||||
app.register_blueprint(web_bp)
|
||||
|
||||
# Load persisted task status after app initialization
|
||||
try:
|
||||
from .shared import (
|
||||
load_task_status_from_file,
|
||||
save_task_status_to_file,
|
||||
set_task_status_file_path,
|
||||
)
|
||||
|
||||
# 只有在 OUTPUT_FOLDER 有效时才启用持久化
|
||||
if hasattr(Config, 'OUTPUT_FOLDER') and Config.OUTPUT_FOLDER:
|
||||
task_status_path = Config.OUTPUT_FOLDER / "task_status.json"
|
||||
set_task_status_file_path(task_status_path)
|
||||
logger.info(f"Task status persistence path set to: {task_status_path}")
|
||||
with app.app_context():
|
||||
load_task_status_from_file()
|
||||
|
||||
import atexit
|
||||
def _save_on_exit():
|
||||
with app.app_context():
|
||||
save_task_status_to_file()
|
||||
atexit.register(_save_on_exit)
|
||||
else:
|
||||
logger.info("Task status persistence will be configured after config is loaded")
|
||||
except Exception as e:
|
||||
print(f"⚠ Failed to setup task persistence: {e}")
|
||||
|
||||
# _get_file_type and _format_response moved to shared.py
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=5000, debug=False)
|
||||
Reference in New Issue
Block a user