| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146 |
- import os
- import sys
- import time
- import signal
- import redis
- from typing import Dict
- from configure import REDIS_CFG
- from logger_setup import setup_app_logger
- from toolkit.cloud_api import CloudApi
- from core_types import BizException
- from utils.booking_outbox import BookingOutbox
- class OrphanTaskSweeper:
- def __init__(self, redis_conf: Dict, logger=None):
- self.logger = logger or setup_app_logger("Sweeper")
- safe_conf = dict(redis_conf)
- safe_conf.setdefault('socket_timeout', 5.0)
- safe_conf.setdefault('socket_connect_timeout', 5.0)
- safe_conf.setdefault('socket_keepalive', True)
- safe_conf.setdefault('health_check_interval', 30)
- self.redis_client = redis.Redis(**safe_conf)
- self._running = True
- self._outbox = BookingOutbox()
- def stop(self, *args):
- if self._running:
- self.logger.info("Stopping Task Sweeper gracefully...")
- self._running = False
- def run_forever(self, interval: float = 3.0):
- self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
- while self._running:
- try:
- self._sweep()
- except redis.ConnectionError as ce:
- self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...")
- time.sleep(5)
- except Exception as e:
- self.logger.error(f"Sweeper loop error: {e}")
-
- # 优雅休眠,支持即时响应退出信号
- step = 0.5
- elapsed = 0.0
- while self._running and elapsed < interval:
- time.sleep(step)
- elapsed += step
- self.logger.info("Task Sweeper stopped.")
- def _sweep(self):
- # 扫描集群中所有 Worker 的 tracking keys
- self._outbox.flush(CloudApi.Instance(), self.redis_client, self.logger.warning)
- tracker_keys = self.redis_client.scan_iter(match="vs:worker:tasks_tracker:*", count=100)
- current_time = time.time()
- for key in tracker_keys:
- if not self._running:
- break
- key_str = key.decode('utf-8') if isinstance(key, bytes) else key
-
- # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
- dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
- if not dead_tasks:
- continue
- for task_id_bytes in dead_tasks:
- if not self._running:
- break
- try:
- task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes)
- self._rescue_task(key_str, task_id)
- except ValueError:
- self.redis_client.zrem(key_str, task_id_bytes)
- def _rescue_task(self, tracker_key: str, task_id: int):
- self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
- try:
- deadline = self.redis_client.zscore(tracker_key, task_id)
- if deadline is None or deadline > time.time():
- return
- # A successful booking must be synchronized, never rebooked.
- if self._outbox.get(task_id) is not None:
- return
- success = self.redis_client.get(self._outbox.marker(task_id))
- if success:
- import json
- CloudApi.Instance().update_vas_task(str(task_id), json.loads(success))
- self.redis_client.zrem(tracker_key, task_id)
- self.redis_client.delete(self._outbox.marker(task_id))
- return
- task_data = None
- try:
- task_data = CloudApi.Instance().get_vas_task(str(task_id))
- except BizException as be:
- if be.code == 40401 or "not found" in be.message.lower():
- self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.")
- self.redis_client.zrem(tracker_key, task_id)
- return
- raise be
-
- if not task_data:
- self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
- self.redis_client.zrem(tracker_key, task_id)
- return
- current_status = task_data.get('status', '')
- if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed', 'failed']:
- self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
- self.redis_client.zrem(tracker_key, task_id)
- return
- # 云端状态还是 processing,证明 Worker 真的死机了,强制归还
- self.logger.info(f"Returning dead task={task_id} to queue.")
- # Cloud reads can take seconds; a worker may have renewed meanwhile.
- deadline = self.redis_client.zscore(tracker_key, task_id)
- if deadline is None or deadline > time.time():
- return
- CloudApi.Instance().return_vas_task_to_queue(task_id)
-
- self.redis_client.zrem(tracker_key, task_id)
- self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.")
- except Exception as e:
- self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.")
- def main():
- logger = setup_app_logger("Sweeper")
- sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger)
- def _signal_handler(signum, frame):
- logger.info(f"Received signal {signum}. Triggering shutdown...")
- sweeper.stop()
- signal.signal(signal.SIGINT, _signal_handler)
- signal.signal(signal.SIGTERM, _signal_handler)
- sweeper.run_forever()
- if __name__ == "__main__":
- main()
|