main_sweeper.py 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146
  1. import os
  2. import sys
  3. import time
  4. import signal
  5. import redis
  6. from typing import Dict
  7. from configure import REDIS_CFG
  8. from logger_setup import setup_app_logger
  9. from toolkit.cloud_api import CloudApi
  10. from core_types import BizException
  11. from utils.booking_outbox import BookingOutbox
  12. class OrphanTaskSweeper:
  13. def __init__(self, redis_conf: Dict, logger=None):
  14. self.logger = logger or setup_app_logger("Sweeper")
  15. safe_conf = dict(redis_conf)
  16. safe_conf.setdefault('socket_timeout', 5.0)
  17. safe_conf.setdefault('socket_connect_timeout', 5.0)
  18. safe_conf.setdefault('socket_keepalive', True)
  19. safe_conf.setdefault('health_check_interval', 30)
  20. self.redis_client = redis.Redis(**safe_conf)
  21. self._running = True
  22. self._outbox = BookingOutbox()
  23. def stop(self, *args):
  24. if self._running:
  25. self.logger.info("Stopping Task Sweeper gracefully...")
  26. self._running = False
  27. def run_forever(self, interval: float = 3.0):
  28. self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
  29. while self._running:
  30. try:
  31. self._sweep()
  32. except redis.ConnectionError as ce:
  33. self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...")
  34. time.sleep(5)
  35. except Exception as e:
  36. self.logger.error(f"Sweeper loop error: {e}")
  37. # 优雅休眠,支持即时响应退出信号
  38. step = 0.5
  39. elapsed = 0.0
  40. while self._running and elapsed < interval:
  41. time.sleep(step)
  42. elapsed += step
  43. self.logger.info("Task Sweeper stopped.")
  44. def _sweep(self):
  45. # 扫描集群中所有 Worker 的 tracking keys
  46. self._outbox.flush(CloudApi.Instance(), self.redis_client, self.logger.warning)
  47. tracker_keys = self.redis_client.scan_iter(match="vs:worker:tasks_tracker:*", count=100)
  48. current_time = time.time()
  49. for key in tracker_keys:
  50. if not self._running:
  51. break
  52. key_str = key.decode('utf-8') if isinstance(key, bytes) else key
  53. # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
  54. dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
  55. if not dead_tasks:
  56. continue
  57. for task_id_bytes in dead_tasks:
  58. if not self._running:
  59. break
  60. try:
  61. task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes)
  62. self._rescue_task(key_str, task_id)
  63. except ValueError:
  64. self.redis_client.zrem(key_str, task_id_bytes)
  65. def _rescue_task(self, tracker_key: str, task_id: int):
  66. self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
  67. try:
  68. deadline = self.redis_client.zscore(tracker_key, task_id)
  69. if deadline is None or deadline > time.time():
  70. return
  71. # A successful booking must be synchronized, never rebooked.
  72. if self._outbox.get(task_id) is not None:
  73. return
  74. success = self.redis_client.get(self._outbox.marker(task_id))
  75. if success:
  76. import json
  77. CloudApi.Instance().update_vas_task(str(task_id), json.loads(success))
  78. self.redis_client.zrem(tracker_key, task_id)
  79. self.redis_client.delete(self._outbox.marker(task_id))
  80. return
  81. task_data = None
  82. try:
  83. task_data = CloudApi.Instance().get_vas_task(str(task_id))
  84. except BizException as be:
  85. if be.code == 40401 or "not found" in be.message.lower():
  86. self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.")
  87. self.redis_client.zrem(tracker_key, task_id)
  88. return
  89. raise be
  90. if not task_data:
  91. self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
  92. self.redis_client.zrem(tracker_key, task_id)
  93. return
  94. current_status = task_data.get('status', '')
  95. if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed', 'failed']:
  96. self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
  97. self.redis_client.zrem(tracker_key, task_id)
  98. return
  99. # 云端状态还是 processing,证明 Worker 真的死机了,强制归还
  100. self.logger.info(f"Returning dead task={task_id} to queue.")
  101. # Cloud reads can take seconds; a worker may have renewed meanwhile.
  102. deadline = self.redis_client.zscore(tracker_key, task_id)
  103. if deadline is None or deadline > time.time():
  104. return
  105. CloudApi.Instance().return_vas_task_to_queue(task_id)
  106. self.redis_client.zrem(tracker_key, task_id)
  107. self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.")
  108. except Exception as e:
  109. self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.")
  110. def main():
  111. logger = setup_app_logger("Sweeper")
  112. sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger)
  113. def _signal_handler(signum, frame):
  114. logger.info(f"Received signal {signum}. Triggering shutdown...")
  115. sweeper.stop()
  116. signal.signal(signal.SIGINT, _signal_handler)
  117. signal.signal(signal.SIGTERM, _signal_handler)
  118. sweeper.run_forever()
  119. if __name__ == "__main__":
  120. main()