import os import sys import time import signal import redis from typing import Dict from configure import REDIS_CFG from logger_setup import setup_app_logger from toolkit.cloud_api import CloudApi from core_types import BizException from utils.booking_outbox import BookingOutbox class OrphanTaskSweeper: def __init__(self, redis_conf: Dict, logger=None): self.logger = logger or setup_app_logger("Sweeper") safe_conf = dict(redis_conf) safe_conf.setdefault('socket_timeout', 5.0) safe_conf.setdefault('socket_connect_timeout', 5.0) safe_conf.setdefault('socket_keepalive', True) safe_conf.setdefault('health_check_interval', 30) self.redis_client = redis.Redis(**safe_conf) self._running = True self._outbox = BookingOutbox() def stop(self, *args): if self._running: self.logger.info("Stopping Task Sweeper gracefully...") self._running = False def run_forever(self, interval: float = 3.0): self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...") while self._running: try: self._sweep() except redis.ConnectionError as ce: self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...") time.sleep(5) except Exception as e: self.logger.error(f"Sweeper loop error: {e}") # 优雅休眠,支持即时响应退出信号 step = 0.5 elapsed = 0.0 while self._running and elapsed < interval: time.sleep(step) elapsed += step self.logger.info("Task Sweeper stopped.") def _sweep(self): # 扫描集群中所有 Worker 的 tracking keys self._outbox.flush(CloudApi.Instance(), self.redis_client, self.logger.warning) tracker_keys = self.redis_client.scan_iter(match="vs:worker:tasks_tracker:*", count=100) current_time = time.time() for key in tracker_keys: if not self._running: break key_str = key.decode('utf-8') if isinstance(key, bytes) else key # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了 dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time) if not dead_tasks: continue for task_id_bytes in dead_tasks: if not self._running: break try: task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes) self._rescue_task(key_str, task_id) except ValueError: self.redis_client.zrem(key_str, task_id_bytes) def _rescue_task(self, tracker_key: str, task_id: int): self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...") try: deadline = self.redis_client.zscore(tracker_key, task_id) if deadline is None or deadline > time.time(): return # A successful booking must be synchronized, never rebooked. if self._outbox.get(task_id) is not None: return success = self.redis_client.get(self._outbox.marker(task_id)) if success: import json CloudApi.Instance().update_vas_task(str(task_id), json.loads(success)) self.redis_client.zrem(tracker_key, task_id) self.redis_client.delete(self._outbox.marker(task_id)) return task_data = None try: task_data = CloudApi.Instance().get_vas_task(str(task_id)) except BizException as be: if be.code == 40401 or "not found" in be.message.lower(): self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.") self.redis_client.zrem(tracker_key, task_id) return raise be if not task_data: self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.") self.redis_client.zrem(tracker_key, task_id) return current_status = task_data.get('status', '') if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed', 'failed']: self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.") self.redis_client.zrem(tracker_key, task_id) return # 云端状态还是 processing,证明 Worker 真的死机了,强制归还 self.logger.info(f"Returning dead task={task_id} to queue.") # Cloud reads can take seconds; a worker may have renewed meanwhile. deadline = self.redis_client.zscore(tracker_key, task_id) if deadline is None or deadline > time.time(): return CloudApi.Instance().return_vas_task_to_queue(task_id) self.redis_client.zrem(tracker_key, task_id) self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.") except Exception as e: self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.") def main(): logger = setup_app_logger("Sweeper") sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger) def _signal_handler(signum, frame): logger.info(f"Received signal {signum}. Triggering shutdown...") sweeper.stop() signal.signal(signal.SIGINT, _signal_handler) signal.signal(signal.SIGTERM, _signal_handler) sweeper.run_forever() if __name__ == "__main__": main()