Sfoglia il codice sorgente

refactor(sweeper): use configure.REDIS_CFG, setup_app_logger and add graceful shutdown; update docker default NODE_ID

jerry 2 mesi fa
parent
commit
c79cef102c
2 ha cambiato i file con 81 aggiunte e 34 eliminazioni
  1. 4 4
      docker-compose.yml
  2. 77 30
      main_sweeper.py

+ 4 - 4
docker-compose.yml

@@ -18,7 +18,7 @@ services:
       - DISPLAY=:99
       - CHROME_BIN=/opt/ungoogled-chromium/chrome
       - MIHOMO_BIN=/bin/mihomo
-      - NODE_ID=${NODE_ID:-node01}
+      - NODE_ID=${NODE_ID:-yecaoyun}
     deploy:
       resources:
         limits:
@@ -42,7 +42,7 @@ services:
       - DISPLAY=:99
       - CHROME_BIN=/opt/ungoogled-chromium/chrome
       - MIHOMO_BIN=/bin/mihomo
-      - NODE_ID=${NODE_ID:-node01}
+      - NODE_ID=${NODE_ID:-yecaoyun}
     deploy:
       resources:
         limits:
@@ -66,7 +66,7 @@ services:
       - DISPLAY=:99
       - CHROME_BIN=/opt/ungoogled-chromium/chrome
       - MIHOMO_BIN=/bin/mihomo
-      - NODE_ID=${NODE_ID:-node01}
+      - NODE_ID=${NODE_ID:-yecaoyun}
     deploy:
       resources:
         limits:
@@ -85,7 +85,7 @@ services:
       - ./logs:/app/logs
     environment:
       - TZ=Asia/Shanghai
-      - NODE_ID=${NODE_ID:-node01}
+      - NODE_ID=${NODE_ID:-yecaoyun}
     deploy:
       resources:
         limits:

+ 77 - 30
main_sweeper.py

@@ -1,26 +1,51 @@
 import os
-import json
+import sys
 import time
+import signal
 import redis
-import logging
 from typing import Dict
+
+from configure import REDIS_CFG
+from logger_setup import setup_app_logger
 from toolkit.vs_cloud_api import VSCloudApi
+from vs_types import BizException
 
-logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] [SWEEPER] %(message)s')
 
 class OrphanTaskSweeper:
-    def __init__(self, redis_conf: Dict):
-        self.redis_client = redis.Redis(**redis_conf)
+    def __init__(self, redis_conf: Dict, logger=None):
+        self.logger = logger or setup_app_logger("Sweeper")
+        safe_conf = dict(redis_conf)
+        safe_conf.setdefault('socket_timeout', 5.0)
+        safe_conf.setdefault('socket_connect_timeout', 5.0)
+        safe_conf.setdefault('socket_keepalive', True)
+        safe_conf.setdefault('health_check_interval', 30)
+        self.redis_client = redis.Redis(**safe_conf)
+        self._running = True
+
+    def stop(self, *args):
+        if self._running:
+            self.logger.info("Stopping Task Sweeper gracefully...")
+            self._running = False
 
-    def run_forever(self):
-        logging.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
-        while True:
+    def run_forever(self, interval: float = 3.0):
+        self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
+        while self._running:
             try:
                 self._sweep()
+            except redis.ConnectionError as ce:
+                self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...")
+                time.sleep(5)
             except Exception as e:
-                logging.error(f"Sweeper loop error: {e}")
+                self.logger.error(f"Sweeper loop error: {e}")
             
-            time.sleep(3)
+            # 优雅休眠,支持即时响应退出信号
+            step = 0.5
+            elapsed = 0.0
+            while self._running and elapsed < interval:
+                time.sleep(step)
+                elapsed += step
+
+        self.logger.info("Task Sweeper stopped.")
 
     def _sweep(self):
         # 扫描集群中所有 Worker 的 tracking keys
@@ -31,51 +56,73 @@ class OrphanTaskSweeper:
         current_time = time.time()
 
         for key in tracker_keys:
+            if not self._running:
+                break
+
             key_str = key.decode('utf-8') if isinstance(key, bytes) else key
             
             # 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
             dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
-            
             if not dead_tasks:
                 continue
 
             for task_id_bytes in dead_tasks:
-                task_id = int(task_id_bytes.decode('utf-8'))
-                self._rescue_task(key_str, task_id)
+                if not self._running:
+                    break
+                try:
+                    task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes)
+                    self._rescue_task(key_str, task_id)
+                except ValueError:
+                    self.redis_client.zrem(key_str, task_id_bytes)
 
     def _rescue_task(self, tracker_key: str, task_id: int):
-        logging.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
+        self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
         try:
-            task_data = VSCloudApi.Instance().get_vas_task(task_id)
+            task_data = None
+            try:
+                task_data = VSCloudApi.Instance().get_vas_task(str(task_id))
+            except BizException as be:
+                if be.code == 40401 or "not found" in be.message.lower():
+                    self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.")
+                    self.redis_client.zrem(tracker_key, task_id)
+                    return
+                raise be
             
             if not task_data:
-                logging.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
+                self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
                 self.redis_client.zrem(tracker_key, task_id)
                 return
 
             current_status = task_data.get('status', '')
-            
-            if current_status in ['cancelled', 'pause' ,'grabbed', 'success', 'completed']:
-                logging.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
+            if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed']:
+                self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
                 self.redis_client.zrem(tracker_key, task_id)
                 return
 
             # 云端状态还是 processing,证明 Worker 真的死机了,强制归还
-            logging.info(f"Returning dead task={task_id} to queue.")
+            self.logger.info(f"Returning dead task={task_id} to queue.")
             VSCloudApi.Instance().return_vas_task_to_queue(task_id)
             
             self.redis_client.zrem(tracker_key, task_id)
-            logging.info(f"✅ Dead task={task_id} successfully rescued and returned.")
+            self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.")
 
         except Exception as e:
-            logging.error(f"Failed to rescue task={task_id}: {e}. Will retry next minute.")
+            self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.")
+
+
+def main():
+    logger = setup_app_logger("Sweeper")
+    sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger)
+
+    def _signal_handler(signum, frame):
+        logger.info(f"Received signal {signum}. Triggering shutdown...")
+        sweeper.stop()
+
+    signal.signal(signal.SIGINT, _signal_handler)
+    signal.signal(signal.SIGTERM, _signal_handler)
+
+    sweeper.run_forever()
+
 
 if __name__ == "__main__":
-    config_path = os.path.join(os.path.dirname(__file__), "config", "config.json")
-    
-    with open(config_path, 'r') as f:
-        config_data = json.load(f)
-        
-    redis_conf = config_data.get('redis', {})
-    sweeper = OrphanTaskSweeper(redis_conf)
-    sweeper.run_forever()
+    main()