@ -16,6 +16,141 @@ from ..app import process_data_async
from . . shared import _format_response , log_performance , logger , ALLOWED_DATA_EXTENSIONS , ALLOWED_CONFIG_EXTENSIONS , allowed_file , update_task_status , TASK_STATUS_PENDING , TASK_STATUS_FAILED
from . . auth import require_api_key
def get_file_extension ( filename ) :
""" 获取文件扩展名，与 allowed_file 函数保持一致 """
if ' . ' not in filename :
return ' '
return ' . ' + filename . rsplit ( ' . ' , 1 ) [ 1 ] . lower ( )
def validate_upload_files ( data_file , config_file = None ) :
""" 验证上传文件并返回详细的错误信息 """
errors = [ ]
# 检查数据文件
if not data_file or data_file . filename == ' ' :
errors . append ( {
' field ' : ' file ' ,
' error ' : ' 数据文件不能为空 ' ,
' details ' : ' 请上传一个有效的Excel文件 '
} )
elif not allowed_file ( data_file . filename , ALLOWED_DATA_EXTENSIONS ) :
errors . append ( {
' field ' : ' file ' ,
' error ' : f ' 不支持的文件类型: { get_file_extension ( data_file . filename ) } ' ,
' details ' : f ' 只支持以下格式: { " , " . join ( ALLOWED_DATA_EXTENSIONS ) } '
} )
# 检查配置文件（如果提供）
if config_file and config_file . filename != ' ' :
if not allowed_file ( config_file . filename , ALLOWED_CONFIG_EXTENSIONS ) :
errors . append ( {
' field ' : ' config ' ,
' error ' : f ' 不支持的配置文件类型: { get_file_extension ( config_file . filename ) } ' ,
' details ' : f ' 只支持以下格式: { " , " . join ( ALLOWED_CONFIG_EXTENSIONS ) } '
} )
return errors
def parse_config_file ( config_file ) :
""" 解析配置文件并提供详细错误信息 """
if not config_file or config_file . filename == ' ' :
# 使用默认配置
try :
default_config_path = Path ( __file__ ) . parent . parent / " gasflux_config.yaml "
with open ( default_config_path , ' r ' , encoding = ' utf-8 ' ) as f :
return yaml . safe_load ( f ) , None
except Exception as e :
return None , f " 加载默认配置文件失败: { str ( e ) } "
try :
config_file . stream . seek ( 0 )
config_text = config_file . read ( ) . decode ( ' utf-8 ' , errors = ' strict ' )
if not config_text . strip ( ) :
return None , " 配置文件为空 "
config_data = yaml . safe_load ( config_text )
# 基本验证
if not isinstance ( config_data , dict ) :
return None , " 配置文件格式错误：应为字典格式 "
# 重置流用于后续保存
config_file . stream = BytesIO ( config_text . encode ( ' utf-8 ' ) )
return config_data , None
except UnicodeDecodeError as e :
return None , f " 配置文件编码错误， { str ( e ) } "
except yaml . YAMLError as e :
return None , f " YAML语法错误: { str ( e ) } "
except Exception as e :
return None , f " 配置文件解析失败: { str ( e ) } "
def save_upload_file ( file_obj , save_path , task_id , file_type = " file " ) :
""" 安全保存上传文件并提供详细错误信息 """
try :
# 检查磁盘空间
save_path . parent . mkdir ( parents = True , exist_ok = True )
# 检查文件大小（ ）
if hasattr ( file_obj , ' content_length ' ) and file_obj . content_length :
max_size = current_app . config . get ( ' MAX_CONTENT_LENGTH ' , 104857600 ) # 100MB
if file_obj . content_length > max_size :
return False , f " { file_type } 文件过大 ( { file_obj . content_length } bytes)，最大限制: { max_size } bytes "
# 保存文件
file_obj . seek ( 0 )
file_obj . save ( str ( save_path ) )
# 验证文件是否成功保存
if not save_path . exists ( ) :
return False , f " { file_type } 文件保存失败：文件不存在 "
if save_path . stat ( ) . st_size == 0 :
return False , f " { file_type } 文件保存失败：文件为空 "
logger . info ( f " Job { task_id } : { file_type } saved successfully - { save_path } ( { save_path . stat ( ) . st_size } bytes) " )
return True , None
except PermissionError :
return False , f " { file_type } 保存失败：没有写入权限 - { save_path . parent } "
except OSError as e :
if e . errno == 28 : # No space left on device
return False , f " { file_type } 保存失败：磁盘空间不足 "
elif e . errno == 36 : # File name too long
return False , f " { file_type } 文件名过长 "
else :
return False , f " { file_type } 保存失败：磁盘错误 ( { e . errno } ) "
except Exception as e :
logger . error ( f " Job { task_id } : Failed to save { file_type } : { str ( e ) } " , exc_info = True )
return False , f " { file_type } 保存失败： { str ( e ) } "
def format_upload_error ( error_code , message , details = None , task_id = None ) :
""" 统一的错误响应格式 """
response = {
' error ' : message ,
' code ' : error_code
}
if details :
response [ ' details ' ] = details
if task_id :
response [ ' task_id ' ] = task_id
# 记录详细错误日志
log_level = ' warning ' if error_code < 500 else ' error '
log_func = getattr ( logger , log_level )
log_func ( f " Upload error [ { error_code } ]: { message } " + ( f " - Details: { details } " if details else " " ) )
return _format_response ( error_code , message , response if details else None )
# Create blueprint
upload_bp = Blueprint ( ' upload ' , __name__ , url_prefix = ' /upload ' )
@ -25,97 +160,80 @@ upload_bp = Blueprint('upload', __name__, url_prefix='/upload')
@log_performance
def upload_file ( ) :
logger . info ( " Received upload request " )
logger . info ( f " Request content length: { request . content_length } bytes " )
# Check if data file is present
# 1. 基础验证
if ' file ' not in request . files :
logger . warning ( " Upload failed: No data file part in request " )
return _format_response ( 400 , " 未找到数据文件部分 " )
return format_upload_error ( 400 , " 未找到数据文件 " , " 请求中缺少 ' file ' 字段 " )
data_file = request . files [ ' file ' ]
config_file = request . files . get ( ' config ' )
# Log file details
logger . info ( f " Data file: { data_file . filename } (size: { getattr ( data_file , ' content_length ' , ' unknown ' ) } bytes) " )
if config_file :
logger . info ( f " Config file: { config_file . filename } (size: { getattr ( config_file , ' content_length ' , ' unknown ' ) } bytes) " )
else :
logger . info ( " No custom config file provided, will use default " )
# 2. 文件验证
validation_errors = validate_upload_files ( data_file , config_file )
if validation_errors :
# 返回第一个错误
error = validation_errors [ 0 ]
return format_upload_error ( 400 , error [ ' error ' ] , error [ ' details ' ] )
if data_file . filename == ' ' :
logger . warning ( " Upload failed: No data file selected (empty filename) " )
return _format_response ( 400 , " 未选择数据文件 " )
if not allowed_file ( data_file . filename , ALLOWED_DATA_EXTENSIONS ) :
logger . warning ( f " Upload failed: Invalid data file type { data_file . filename } - allowed: { ALLOWED_DATA_EXTENSIONS } " )
return _format_response ( 400 , " 无效的数据文件类型。只允许 .xlsx 和 .xls 格式。 " )
# Generate unique job ID
# 3. 生成任务ID
task_id = str ( uuid . uuid4 ( ) )
logger . info ( f " Generated job ID: { task_id } " )
# 1) Parse config content (parse in memory without saving first)
if config_file and config_file . filename != ' ' :
if not allowed_file ( config_file . filename , ALLOWED_CONFIG_EXTENSIONS ) :
return _ format_response ( 400 , " 无效的 配置文件类型。只允许 .yaml 和 .yml 格式。 " )
config_file . stream . seek ( 0 )
config_text = config_file . read ( ) . decode ( ' utf-8 ' , errors = ' ignore ' )
try :
active_config = yaml . safe_load ( config_text )
except Exception :
return _format_response ( 400 , " 配置文件解析失败 " )
# Reset stream for saving
config_file . stream = BytesIO ( config_text . encode ( ' utf-8 ' ) )
else :
default_config_path = Path ( __file__ ) . parent . parent / " gasflux_config.yaml "
with open ( default_config_path , ' r ' , encoding = ' utf-8 ' ) as f :
active_config = yaml . safe_load ( f )
# 4. 解析配置
active_config , config_error = parse_config_file ( config_file )
if config_error :
return format_upload_error ( 400 , " 配置文件错误 " , config_error , task_id )
# 2) Create job directories based on INI configuration
upload_base = Path ( current_app . config [ ' UPLOAD_FOLDER ' ] )
output _base= Path ( current_app . config [ ' OUTPUT _FOLDER' ] )
job_upload_dir = upload_base / task_id
job_output _dir = output _base/ task_id
job_upload_dir . mkdir ( parents = True , exist_ok = True )
job_output_dir . mkdir ( parents = True , exist_ok = True )
logger . info ( f " Job { task_id } : Created directories - Upload: { job_upload_dir } , Output: { job_output_dir } " )
# 5. 创建目录
try :
upload _base= Path ( current_app . config [ ' UPLOAD _FOLDER' ] )
output_base = Path ( current_app . config [ ' OUTPUT_FOLDER ' ] )
job_upload _dir = upload _base/ task_id
job_output_dir = output_base / task_id
# 3) Save data file to job_upload_dir
job_upload_dir . mkdir ( parents = True , exist_ok = True )
job_output_dir . mkdir ( parents = True , exist_ok = True )
logger . info ( f " Job { task_id } : Directories created successfully " )
except Exception as e :
return format_upload_error ( 500 , " 目录创建失败 " , str ( e ) , task_id )
# 6. 保存数据文件
data_filename = secure_filename ( data_file . filename )
data_path = job_upload_dir / data_filename
try :
data_file . seek ( 0 )
data_file . save ( str ( data_path ) )
logger . info ( f " Job { task_id } : Data file saved successfully - Path: { data_path } " )
except Exception as e :
logger . error ( f " Job { task_id } : Failed to save data file { data_filename } : { str ( e ) } " )
return _format_response ( 500 , " 保存数据文件失败 " )
# 4) Save config file to job_upload_dir
success , error_msg = save_upload_file ( data_file , data_path , task_id , " 数据文件 " )
if not success :
return format_upload_error ( 500 , error_msg , task_id = task_id )
# 7. 保存配置文件
if config_file and config_file . filename != ' ' :
config_filename = secure_filename ( config_file . filename )
config_path = job_upload_dir / config_filename
try :
config_file . seek ( 0 )
config_file . save ( str ( config_path ) )
active_config_path = config_path
logger . info ( f " Job { task_id } : Custom config saved successfully - Path: { config_path } " )
except Exception as e :
logger . error ( f " Job { task_id } : Failed to save config file { config_filename } : { str ( e ) } " )
return _format_response ( 500 , " 保存配置文件失败 " )
else :
# Copy default config for record keeping
config_path = job_upload_dir / " config.yaml "
with open ( config_path , ' w ' , encoding = ' utf-8 ' ) as f :
yaml . safe_dump ( active_config , f , allow_unicode = True )
active_config_path = config_path
logger . info ( f " Job { task_id } : Default config saved for record - Path: { config_path } " )
# Initialize task status
update_task_status ( task_id , TASK_STATUS_PENDING , " 任务已加入处理队列 " , output_dir = str ( job_output_dir ) )
logger . info ( f " Job { task_id } : Task status initialized as PENDING " )
success , error_msg = save_upload_file ( config_file , config_path , task_id , " 配置文件 " )
if not success :
return format_upload_error ( 500 , error_msg , task_id = task_id )
else :
# 保存默认配置
config_path = job_upload_dir / " config.yaml "
try :
with open ( config_path , ' w ' , encoding = ' utf-8 ' ) as f :
yaml . safe_dump ( active_config , f , allow_unicode = True )
active_config_path = config_path
logger . info ( f " Job { task_id } : Default config saved " )
except Exception as e :
return format_upload_error ( 500 , f " 保存默认配置失败: { str ( e ) } " , task_id = task_id )
# Start background processing
# 8. 初始化任务状态
try :
update_task_status ( task_id , TASK_STATUS_PENDING , " 任务已加入处理队列 " , output_dir = str ( job_output_dir ) )
except Exception as e :
logger . error ( f " Job { task_id } : Failed to update task status: { str ( e ) } " )
# 继续处理，不返回错误
# 9. 启动后台处理
try :
thread = threading . Thread (
target = process_data_async ,
@ -123,15 +241,16 @@ def upload_file():
)
thread . daemon = True
thread . start ( )
logger . info ( f " Job { task_id } : Background processing thread started successfully " )
logger . info ( f " Job { task_id } : Background processing started " )
except Exception as e :
logger . error ( f " Job { task_id } : Failed to start background processing thread: { str ( e ) } " )
logger . error ( f " Job { task_id } : Failed to start processing thread: { str ( e ) } " , exc_info = True
update_task_status ( task_id , TASK_STATUS_FAILED , error = str ( e ) )
return _ format_response ( 500 , " 启动处理失败 " )
return format_upload_error ( 500 , " 启动处理线程 失败 " , str ( e ) , task_id
logger . info ( f " Job { task_id } : Upload process completed successfully, returning job ID to client " )
# 10. 返回成功响应
return _format_response ( 202 , " 任务已接受并加入处理队列 " , {
" status " : " accepted " ,
" task_id " : task_id ,
" task_status_url " : f " /task/ { task_id } "
" task_status_url " : f " /task/ { task_id } " ,
" message " : f " 数据文件 ' { data_filename } ' 已上传，开始处理 "
} )
