main_sweeper.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. import os
  2. import sys
  3. import time
  4. import signal
  5. import redis
  6. from typing import Dict
  7. from configure import REDIS_CFG
  8. from logger_setup import setup_app_logger
  9. from toolkit.cloud_api import CloudApi
  10. from core_types import BizException
  11. from utils.booking_outbox import BookingOutbox
  12. class OrphanTaskSweeper:
  13. def __init__(self, redis_conf: Dict, logger=None):
  14. self.logger = logger or setup_app_logger("Sweeper")
  15. safe_conf = dict(redis_conf)
  16. safe_conf.setdefault('socket_timeout', 5.0)
  17. safe_conf.setdefault('socket_connect_timeout', 5.0)
  18. safe_conf.setdefault('socket_keepalive', True)
  19. safe_conf.setdefault('health_check_interval', 30)
  20. self.redis_client = redis.Redis(**safe_conf)
  21. self._running = True
  22. self._outbox = BookingOutbox()
  23. def stop(self, *args):
  24. if self._running:
  25. self.logger.info("Stopping Task Sweeper gracefully...")
  26. self._running = False
  27. def run_forever(self, interval: float = 3.0):
  28. from toolkit.notifier import start_notification_worker
  29. start_notification_worker()
  30. self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
  31. while self._running:
  32. try:
  33. self._sweep()
  34. except redis.ConnectionError as ce:
  35. self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...")
  36. time.sleep(5)
  37. except Exception as e:
  38. self.logger.error(f"Sweeper loop error: {e}")
  39. # 优雅休眠,支持即时响应退出信号
  40. step = 0.5
  41. elapsed = 0.0
  42. while self._running and elapsed < interval:
  43. time.sleep(step)
  44. elapsed += step
  45. self.logger.info("Task Sweeper stopped.")
  46. def _sweep(self):
  47. # 扫描集群中所有 Worker 的 tracking keys
  48. self._outbox.flush(CloudApi.Instance(), self.redis_client, self.logger.warning)
  49. tracker_keys = self.redis_client.scan_iter(match="vs:worker:tasks_tracker:*", count=100)
  50. current_time = time.time()
  51. for key in tracker_keys:
  52. if not self._running:
  53. break
  54. key_str = key.decode('utf-8') if isinstance(key, bytes) else key
  55. # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
  56. dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
  57. if not dead_tasks:
  58. continue
  59. for task_id_bytes in dead_tasks:
  60. if not self._running:
  61. break
  62. try:
  63. task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes)
  64. self._rescue_task(key_str, task_id)
  65. except ValueError:
  66. self.redis_client.zrem(key_str, task_id_bytes)
  67. def _rescue_task(self, tracker_key: str, task_id: int):
  68. self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
  69. try:
  70. deadline = self.redis_client.zscore(tracker_key, task_id)
  71. if deadline is None or deadline > time.time():
  72. return
  73. # A successful booking must be synchronized, never rebooked.
  74. if self._outbox.get(task_id) is not None:
  75. return
  76. success = self.redis_client.get(self._outbox.marker(task_id))
  77. if success:
  78. import json
  79. self._outbox.put(task_id, tracker_key, json.loads(success))
  80. self._outbox.flush(CloudApi.Instance(), self.redis_client, self.logger.warning)
  81. return
  82. task_data = None
  83. try:
  84. task_data = CloudApi.Instance().get_vas_task(str(task_id))
  85. except BizException as be:
  86. if be.code == 40401 or "not found" in be.message.lower():
  87. self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.")
  88. self.redis_client.zrem(tracker_key, task_id)
  89. return
  90. raise be
  91. if not task_data:
  92. self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
  93. self.redis_client.zrem(tracker_key, task_id)
  94. return
  95. current_status = task_data.get('status', '')
  96. if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed', 'failed']:
  97. self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
  98. self.redis_client.zrem(tracker_key, task_id)
  99. return
  100. # 云端状态还是 processing,证明 Worker 真的死机了,强制归还
  101. self.logger.info(f"Returning dead task={task_id} to queue.")
  102. # Cloud reads can take seconds; a worker may have renewed meanwhile.
  103. deadline = self.redis_client.zscore(tracker_key, task_id)
  104. if deadline is None or deadline > time.time():
  105. return
  106. CloudApi.Instance().return_vas_task_to_queue(task_id)
  107. self.redis_client.zrem(tracker_key, task_id)
  108. self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.")
  109. except Exception as e:
  110. self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.")
  111. def main():
  112. logger = setup_app_logger("Sweeper")
  113. sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger)
  114. def _signal_handler(signum, frame):
  115. logger.info(f"Received signal {signum}. Triggering shutdown...")
  116. sweeper.stop()
  117. signal.signal(signal.SIGINT, _signal_handler)
  118. signal.signal(signal.SIGTERM, _signal_handler)
  119. sweeper.run_forever()
  120. if __name__ == "__main__":
  121. main()