revert: 回退克里金多线程并发,恢复串行处理

并发实测无提速(32GB 内存下单个克里金已用 5-6GB,两个并发
竞争内存带宽导致实际耗时相同),回退为简单串行循环。
This commit is contained in:
duxin
2026-07-28 16:53:15 +08:00
parent aa5d9d9392
commit 1efe10adb6
2 changed files with 51 additions and 160 deletions

View File

@ -243,79 +243,32 @@ class Step11MapHandler(BaseStepHandler):
context.notify('step11_map', 'warning',
f'共享上下文预计算失败: {e},回退逐个处理')
# ── 并发生成 ──
# 克里金插值内部为 numpy/scipy 运算(释放 GIL),
# 使用 ThreadPoolExecutor 并发处理多个 CSV,大幅缩短总耗时。
# 注意:不使用 ProcessPoolExecutor(Windows spawn 会导致死锁)。
_max_workers = int(config.get('kriging_workers', 2))
_max_workers = max(1, min(_max_workers, total, os.cpu_count() or 4))
# ── 串行生成 ──
# 注:克里金是内存密集型运算(32GB 下单个用 5-6GB),
# 多线程并发竞争内存带宽,实际无提速,因此保持串行。
generated: List[str] = []
errors: Dict[str, str] = {}
if _max_workers > 1 and total > 1:
# ★ 主线程预先设置 matplotlib Agg 后端(避免多线程竞争)
import matplotlib
context.notify('step11_map', 'info',
f'串行生成 {total} 张专题图(克里金自适应分块)')
for idx, csv_p in enumerate(csv_paths):
percent = int(idx / total * 100)
context.notify('step11_map', 'info',
f'专题图 [{idx+1}/{total}]: {Path(csv_p).name}')
global_event_bus.publish('ProgressUpdate', {
'percentage': percent,
'message': f'Step11: {idx+1}/{total} {Path(csv_p).stem}',
})
try:
matplotlib.use('Agg', force=True)
except Exception:
pass
import concurrent.futures
context.notify('step11_map', 'info',
f'并发生成 {total} 张专题图({_max_workers} 线程并行)')
completed = 0
with concurrent.futures.ThreadPoolExecutor(
max_workers=_max_workers) as executor:
future_map = {
executor.submit(
_process_one_map, csv_p, base_kwargs, output_dir
): csv_p
for csv_p in csv_paths
}
for future in concurrent.futures.as_completed(future_map):
csv_p = future_map[future]
completed += 1
try:
result_path, _ = future.result()
generated.append(result_path)
context.notify('step11_map', 'info',
f'专题图 [{completed}/{total}] ✓: '
f'{Path(csv_p).name}')
except Exception as e:
errors[csv_p] = str(e)
context.notify('step11_map', 'warning',
f'专题图 [{completed}/{total}] ✗: '
f'{Path(csv_p).name} — {e}')
percent = int(completed / total * 100)
global_event_bus.publish('ProgressUpdate', {
'percentage': percent,
'message': f'Step11: {completed}/{total} '
f'{Path(csv_p).stem}',
})
else:
context.notify('step11_map', 'info',
f'顺序生成 {total} 张专题图(局部 Kriging 自适应分块)')
for idx, csv_p in enumerate(csv_paths):
percent = int(idx / total * 100)
context.notify('step11_map', 'info',
f'专题图 [{idx+1}/{total}]: {Path(csv_p).name}')
global_event_bus.publish('ProgressUpdate', {
'percentage': percent,
'message': f'Step11: {idx+1}/{total} {Path(csv_p).stem}',
})
try:
result_path, _ = _process_one_map(csv_p, base_kwargs, output_dir)
generated.append(result_path)
except Exception as e:
errors[csv_p] = str(e)
context.notify('step11_map', 'warning',
f'专题图 FAIL: {Path(csv_p).name} — {e}')
result_path, _ = _process_one_map(csv_p, base_kwargs, output_dir)
generated.append(result_path)
except Exception as e:
errors[csv_p] = str(e)
context.notify('step11_map', 'warning',
f'专题图 FAIL: {Path(csv_p).name} — {e}')
step_end_time = time.time()
elapsed = step_end_time - step_start_time