feat: 专题图克里金插值支持多线程并发
- ThreadPoolExecutor 替代串行 for 循环,默认 2 线程并发 - 克里金内部 numpy/scipy 运算释放 GIL,线程并行有效 - 通过 kriging_workers 配置项控制并发数(默认 2,设为 1 回退串行) - 主线程预先设置 matplotlib Agg 后端,避免多线程竞争 - 保留串行路径作为 fallback(单 CSV 或 workers=1 时)
This commit is contained in:
@ -243,32 +243,79 @@ class Step11MapHandler(BaseStepHandler):
|
|||||||
context.notify('step11_map', 'warning',
|
context.notify('step11_map', 'warning',
|
||||||
f'共享上下文预计算失败: {e},回退逐个处理')
|
f'共享上下文预计算失败: {e},回退逐个处理')
|
||||||
|
|
||||||
# ── 顺序生成(避免 Windows spawn 下 ProcessPoolExecutor 死锁)──
|
# ── 并发生成 ──
|
||||||
# 局部 Kriging 内部已做 16 块顺序分块,每块 ~20-30s,
|
# 克里金插值内部为 numpy/scipy 运算(释放 GIL),
|
||||||
# 每张图约 5-8 分钟。64 张 ≈ 5-8 小时,但进度完全透明可见。
|
# 使用 ThreadPoolExecutor 并发处理多个 CSV,大幅缩短总耗时。
|
||||||
|
# 注意:不使用 ProcessPoolExecutor(Windows spawn 会导致死锁)。
|
||||||
|
_max_workers = int(config.get('kriging_workers', 2))
|
||||||
|
_max_workers = max(1, min(_max_workers, total, os.cpu_count() or 4))
|
||||||
|
|
||||||
generated: List[str] = []
|
generated: List[str] = []
|
||||||
errors: Dict[str, str] = {}
|
errors: Dict[str, str] = {}
|
||||||
|
|
||||||
context.notify('step11_map', 'info',
|
if _max_workers > 1 and total > 1:
|
||||||
f'顺序生成 {total} 张专题图(局部 Kriging 自适应分块)')
|
# ★ 主线程预先设置 matplotlib Agg 后端(避免多线程竞争)
|
||||||
|
import matplotlib
|
||||||
for idx, csv_p in enumerate(csv_paths):
|
|
||||||
percent = int(idx / total * 100)
|
|
||||||
context.notify('step11_map', 'info',
|
|
||||||
f'专题图 [{idx+1}/{total}]: {Path(csv_p).name}')
|
|
||||||
|
|
||||||
global_event_bus.publish('ProgressUpdate', {
|
|
||||||
'percentage': percent,
|
|
||||||
'message': f'Step11: {idx+1}/{total} {Path(csv_p).stem}',
|
|
||||||
})
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result_path, _ = _process_one_map(csv_p, base_kwargs, output_dir)
|
matplotlib.use('Agg', force=True)
|
||||||
generated.append(result_path)
|
except Exception:
|
||||||
except Exception as e:
|
pass
|
||||||
errors[csv_p] = str(e)
|
|
||||||
context.notify('step11_map', 'warning',
|
import concurrent.futures
|
||||||
f'专题图 FAIL: {Path(csv_p).name} — {e}')
|
context.notify('step11_map', 'info',
|
||||||
|
f'并发生成 {total} 张专题图({_max_workers} 线程并行)')
|
||||||
|
|
||||||
|
completed = 0
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(
|
||||||
|
max_workers=_max_workers) as executor:
|
||||||
|
future_map = {
|
||||||
|
executor.submit(
|
||||||
|
_process_one_map, csv_p, base_kwargs, output_dir
|
||||||
|
): csv_p
|
||||||
|
for csv_p in csv_paths
|
||||||
|
}
|
||||||
|
for future in concurrent.futures.as_completed(future_map):
|
||||||
|
csv_p = future_map[future]
|
||||||
|
completed += 1
|
||||||
|
try:
|
||||||
|
result_path, _ = future.result()
|
||||||
|
generated.append(result_path)
|
||||||
|
context.notify('step11_map', 'info',
|
||||||
|
f'专题图 [{completed}/{total}] ✓: '
|
||||||
|
f'{Path(csv_p).name}')
|
||||||
|
except Exception as e:
|
||||||
|
errors[csv_p] = str(e)
|
||||||
|
context.notify('step11_map', 'warning',
|
||||||
|
f'专题图 [{completed}/{total}] ✗: '
|
||||||
|
f'{Path(csv_p).name} — {e}')
|
||||||
|
|
||||||
|
percent = int(completed / total * 100)
|
||||||
|
global_event_bus.publish('ProgressUpdate', {
|
||||||
|
'percentage': percent,
|
||||||
|
'message': f'Step11: {completed}/{total} '
|
||||||
|
f'{Path(csv_p).stem}',
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
context.notify('step11_map', 'info',
|
||||||
|
f'顺序生成 {total} 张专题图(局部 Kriging 自适应分块)')
|
||||||
|
|
||||||
|
for idx, csv_p in enumerate(csv_paths):
|
||||||
|
percent = int(idx / total * 100)
|
||||||
|
context.notify('step11_map', 'info',
|
||||||
|
f'专题图 [{idx+1}/{total}]: {Path(csv_p).name}')
|
||||||
|
|
||||||
|
global_event_bus.publish('ProgressUpdate', {
|
||||||
|
'percentage': percent,
|
||||||
|
'message': f'Step11: {idx+1}/{total} {Path(csv_p).stem}',
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
|
result_path, _ = _process_one_map(csv_p, base_kwargs, output_dir)
|
||||||
|
generated.append(result_path)
|
||||||
|
except Exception as e:
|
||||||
|
errors[csv_p] = str(e)
|
||||||
|
context.notify('step11_map', 'warning',
|
||||||
|
f'专题图 FAIL: {Path(csv_p).name} — {e}')
|
||||||
|
|
||||||
step_end_time = time.time()
|
step_end_time = time.time()
|
||||||
elapsed = step_end_time - step_start_time
|
elapsed = step_end_time - step_start_time
|
||||||
|
|||||||
Reference in New Issue
Block a user