fix: 大尺度影像处理超时 — 心跳线程 + 日志复位看门狗 + 延长至1小时
问题: 用户处理 6522x13215x150 大影像,step3 Goodman 逐波段 C/NumPy 运算耗时超过 10 分钟,期间无 Qt 信号发出,看门狗误判 为假死并强制终止。 修复 (三层防护): 1. WorkerThread 心跳线程: - 新增独立 daemon 线程,每 30s 发射 log_message 信号 - PyQt 信号线程安全,跨线程 emit 自动排队到主线程 - 确保长时间 C 运算期间看门狗持续收到保活信号 - run() finally 中自动停止心跳 2. 日志消息也复位看门狗: _on_log_message() 现在与 _on_progress_update() 一样更新 _last_progress_time —— 任何来自 Worker 的通讯都代表存活 3. 超时阈值 600s → 3600s (1小时): 大尺度影像处理 1 小时足够覆盖极端场景
This commit is contained in:
@ -492,7 +492,14 @@ class PipelineExecutor(QObject):
|
|||||||
# ═══════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════
|
||||||
|
|
||||||
def _on_log_message(self, message: str, level: str):
|
def _on_log_message(self, message: str, level: str):
|
||||||
"""WorkerThread 日志 → EventBus LogMessage 事件。"""
|
"""WorkerThread 日志 → EventBus LogMessage 事件。
|
||||||
|
|
||||||
|
同时重置看门狗计时器:任何来自 Worker 的日志消息都代表 Worker 仍然存活。
|
||||||
|
这解决了长时间运行的 C/NumPy 计算(如 Goodman 逐波段处理)期间
|
||||||
|
因无进度更新而被误杀的问题。
|
||||||
|
"""
|
||||||
|
import time
|
||||||
|
self._last_progress_time = time.time()
|
||||||
global_event_bus.publish('LogMessage', {
|
global_event_bus.publish('LogMessage', {
|
||||||
'message': message,
|
'message': message,
|
||||||
'level': level,
|
'level': level,
|
||||||
@ -541,14 +548,14 @@ class PipelineExecutor(QObject):
|
|||||||
|
|
||||||
判定逻辑:
|
判定逻辑:
|
||||||
- Worker 已不在运行(自然结束/被强杀) → 停掉看门狗
|
- Worker 已不在运行(自然结束/被强杀) → 停掉看门狗
|
||||||
- Worker 仍在运行 + 上次进度距今 > 600 秒 → 判定为假死/死锁,
|
- Worker 仍在运行 + 上次通讯距今 > 3600 秒 → 判定为假死/死锁,
|
||||||
先尝试 WorkerThread.stop() 优雅停止(5 秒超时),
|
先尝试 WorkerThread.stop() 优雅停止(5 秒超时),
|
||||||
若优雅停止也超时则 fallback 到 terminate()
|
若优雅停止也超时则 fallback 到 terminate()
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
if self._worker and self._worker.isRunning():
|
if self._worker and self._worker.isRunning():
|
||||||
if time.time() - self._last_progress_time > 600:
|
if time.time() - self._last_progress_time > 3600:
|
||||||
self._log_message("[错误] 后台任务响应超时(超过10分钟无响应),尝试优雅停止...", "error")
|
self._log_message("[错误] 后台任务响应超时(超过1小时无任何通讯),尝试优雅停止...", "error")
|
||||||
# 先尝试通过 cancel + wait 优雅停止
|
# 先尝试通过 cancel + wait 优雅停止
|
||||||
if hasattr(self._worker, 'stop'):
|
if hasattr(self._worker, 'stop'):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@ -218,6 +218,9 @@ class WorkerThread(QThread):
|
|||||||
k for k in self.config.keys() if k not in self.skip_list
|
k for k in self.config.keys() if k not in self.skip_list
|
||||||
]) if self.mode == 'full' else 1
|
]) if self.mode == 'full' else 1
|
||||||
|
|
||||||
|
# ★ 心跳线程:防止长时间 C/NumPy 运算期间看门狗超时
|
||||||
|
self._heartbeat_stop = None # threading.Event
|
||||||
|
|
||||||
def pipeline_callback(self, step_name, status, message=""):
|
def pipeline_callback(self, step_name, status, message=""):
|
||||||
"""Pipeline回调函数,用于接收步骤状态"""
|
"""Pipeline回调函数,用于接收步骤状态"""
|
||||||
if status == "start":
|
if status == "start":
|
||||||
@ -277,6 +280,23 @@ class WorkerThread(QThread):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
mpl_prev = None
|
mpl_prev = None
|
||||||
|
|
||||||
|
# ★ 启动心跳线程:每 30s 向主线程发送一次保活信号
|
||||||
|
# 防止 Goodman/Kutser 等逐波段 C 运算期间因无 Qt 信号而被看门狗误杀
|
||||||
|
import threading as _threading
|
||||||
|
self._heartbeat_stop = _threading.Event()
|
||||||
|
_heartbeat_interval = 30.0
|
||||||
|
|
||||||
|
def _heartbeat_loop():
|
||||||
|
while not self._heartbeat_stop.is_set():
|
||||||
|
self._heartbeat_stop.wait(_heartbeat_interval)
|
||||||
|
if not self._heartbeat_stop.is_set():
|
||||||
|
# 心跳不打印可见日志(避免刷屏),但必须发射信号以重置看门狗
|
||||||
|
self.log_message.emit("[心跳] 后台处理中...", "info")
|
||||||
|
|
||||||
|
_heartbeat_thread = _threading.Thread(target=_heartbeat_loop, daemon=True)
|
||||||
|
_heartbeat_thread.start()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# ★ 终端即时反馈
|
# ★ 终端即时反馈
|
||||||
print(f"\n[WorkerThread] 后台线程启动 (mode={self.mode}, work_dir={self.work_dir})")
|
print(f"\n[WorkerThread] 后台线程启动 (mode={self.mode}, work_dir={self.work_dir})")
|
||||||
@ -354,6 +374,10 @@ class WorkerThread(QThread):
|
|||||||
self.finished.emit(False, f"后台线程崩溃: {e}\n\n{full_tb}")
|
self.finished.emit(False, f"后台线程崩溃: {e}\n\n{full_tb}")
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
|
# ── 停止心跳线程 ──
|
||||||
|
if self._heartbeat_stop is not None:
|
||||||
|
self._heartbeat_stop.set()
|
||||||
|
|
||||||
# ── 恢复 Matplotlib 后端 ──
|
# ── 恢复 Matplotlib 后端 ──
|
||||||
if mpl_prev:
|
if mpl_prev:
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user