main_sweeper.py 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128
  1. import os
  2. import sys
  3. import time
  4. import signal
  5. import redis
  6. from typing import Dict
  7. from configure import REDIS_CFG
  8. from logger_setup import setup_app_logger
  9. from toolkit.cloud_api import CloudApi
  10. from core_types import BizException
  11. class OrphanTaskSweeper:
  12. def __init__(self, redis_conf: Dict, logger=None):
  13. self.logger = logger or setup_app_logger("Sweeper")
  14. safe_conf = dict(redis_conf)
  15. safe_conf.setdefault('socket_timeout', 5.0)
  16. safe_conf.setdefault('socket_connect_timeout', 5.0)
  17. safe_conf.setdefault('socket_keepalive', True)
  18. safe_conf.setdefault('health_check_interval', 30)
  19. self.redis_client = redis.Redis(**safe_conf)
  20. self._running = True
  21. def stop(self, *args):
  22. if self._running:
  23. self.logger.info("Stopping Task Sweeper gracefully...")
  24. self._running = False
  25. def run_forever(self, interval: float = 3.0):
  26. self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
  27. while self._running:
  28. try:
  29. self._sweep()
  30. except redis.ConnectionError as ce:
  31. self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...")
  32. time.sleep(5)
  33. except Exception as e:
  34. self.logger.error(f"Sweeper loop error: {e}")
  35. # 优雅休眠,支持即时响应退出信号
  36. step = 0.5
  37. elapsed = 0.0
  38. while self._running and elapsed < interval:
  39. time.sleep(step)
  40. elapsed += step
  41. self.logger.info("Task Sweeper stopped.")
  42. def _sweep(self):
  43. # 扫描集群中所有 Worker 的 tracking keys
  44. tracker_keys = self.redis_client.keys("vs:worker:tasks_tracker:*")
  45. if not tracker_keys:
  46. return
  47. current_time = time.time()
  48. for key in tracker_keys:
  49. if not self._running:
  50. break
  51. key_str = key.decode('utf-8') if isinstance(key, bytes) else key
  52. # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
  53. dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
  54. if not dead_tasks:
  55. continue
  56. for task_id_bytes in dead_tasks:
  57. if not self._running:
  58. break
  59. try:
  60. task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes)
  61. self._rescue_task(key_str, task_id)
  62. except ValueError:
  63. self.redis_client.zrem(key_str, task_id_bytes)
  64. def _rescue_task(self, tracker_key: str, task_id: int):
  65. self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
  66. try:
  67. task_data = None
  68. try:
  69. task_data = CloudApi.Instance().get_vas_task(str(task_id))
  70. except BizException as be:
  71. if be.code == 40401 or "not found" in be.message.lower():
  72. self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.")
  73. self.redis_client.zrem(tracker_key, task_id)
  74. return
  75. raise be
  76. if not task_data:
  77. self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
  78. self.redis_client.zrem(tracker_key, task_id)
  79. return
  80. current_status = task_data.get('status', '')
  81. if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed']:
  82. self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
  83. self.redis_client.zrem(tracker_key, task_id)
  84. return
  85. # 云端状态还是 processing,证明 Worker 真的死机了,强制归还
  86. self.logger.info(f"Returning dead task={task_id} to queue.")
  87. CloudApi.Instance().return_vas_task_to_queue(task_id)
  88. self.redis_client.zrem(tracker_key, task_id)
  89. self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.")
  90. except Exception as e:
  91. self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.")
  92. def main():
  93. logger = setup_app_logger("Sweeper")
  94. sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger)
  95. def _signal_handler(signum, frame):
  96. logger.info(f"Received signal {signum}. Triggering shutdown...")
  97. sweeper.stop()
  98. signal.signal(signal.SIGINT, _signal_handler)
  99. signal.signal(signal.SIGTERM, _signal_handler)
  100. sweeper.run_forever()
  101. if __name__ == "__main__":
  102. main()