import os import sys import time import signal import redis from typing import Dict from configure import REDIS_CFG from logger_setup import setup_app_logger from toolkit.vs_cloud_api import VSCloudApi from vs_types import BizException class OrphanTaskSweeper: def __init__(self, redis_conf: Dict, logger=None): self.logger = logger or setup_app_logger("Sweeper") safe_conf = dict(redis_conf) safe_conf.setdefault('socket_timeout', 5.0) safe_conf.setdefault('socket_connect_timeout', 5.0) safe_conf.setdefault('socket_keepalive', True) safe_conf.setdefault('health_check_interval', 30) self.redis_client = redis.Redis(**safe_conf) self._running = True def stop(self, *args): if self._running: self.logger.info("Stopping Task Sweeper gracefully...") self._running = False def run_forever(self, interval: float = 3.0): self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...") while self._running: try: self._sweep() except redis.ConnectionError as ce: self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...") time.sleep(5) except Exception as e: self.logger.error(f"Sweeper loop error: {e}") # 优雅休眠,支持即时响应退出信号 step = 0.5 elapsed = 0.0 while self._running and elapsed < interval: time.sleep(step) elapsed += step self.logger.info("Task Sweeper stopped.") def _sweep(self): # 扫描集群中所有 Worker 的 tracking keys tracker_keys = self.redis_client.keys("vs:worker:tasks_tracker:*") if not tracker_keys: return current_time = time.time() for key in tracker_keys: if not self._running: break key_str = key.decode('utf-8') if isinstance(key, bytes) else key # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了 dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time) if not dead_tasks: continue for task_id_bytes in dead_tasks: if not self._running: break try: task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes) self._rescue_task(key_str, task_id) except ValueError: self.redis_client.zrem(key_str, task_id_bytes) def _rescue_task(self, tracker_key: str, task_id: int): self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...") try: task_data = None try: task_data = VSCloudApi.Instance().get_vas_task(str(task_id)) except BizException as be: if be.code == 40401 or "not found" in be.message.lower(): self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.") self.redis_client.zrem(tracker_key, task_id) return raise be if not task_data: self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.") self.redis_client.zrem(tracker_key, task_id) return current_status = task_data.get('status', '') if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed']: self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.") self.redis_client.zrem(tracker_key, task_id) return # 云端状态还是 processing,证明 Worker 真的死机了,强制归还 self.logger.info(f"Returning dead task={task_id} to queue.") VSCloudApi.Instance().return_vas_task_to_queue(task_id) self.redis_client.zrem(tracker_key, task_id) self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.") except Exception as e: self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.") def main(): logger = setup_app_logger("Sweeper") sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger) def _signal_handler(signum, frame): logger.info(f"Received signal {signum}. Triggering shutdown...") sweeper.stop() signal.signal(signal.SIGINT, _signal_handler) signal.signal(signal.SIGTERM, _signal_handler) sweeper.run_forever() if __name__ == "__main__": main()