|
@@ -1,26 +1,51 @@
|
|
|
import os
|
|
import os
|
|
|
-import json
|
|
|
|
|
|
|
+import sys
|
|
|
import time
|
|
import time
|
|
|
|
|
+import signal
|
|
|
import redis
|
|
import redis
|
|
|
-import logging
|
|
|
|
|
from typing import Dict
|
|
from typing import Dict
|
|
|
|
|
+
|
|
|
|
|
+from configure import REDIS_CFG
|
|
|
|
|
+from logger_setup import setup_app_logger
|
|
|
from toolkit.vs_cloud_api import VSCloudApi
|
|
from toolkit.vs_cloud_api import VSCloudApi
|
|
|
|
|
+from vs_types import BizException
|
|
|
|
|
|
|
|
-logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] [SWEEPER] %(message)s')
|
|
|
|
|
|
|
|
|
|
class OrphanTaskSweeper:
|
|
class OrphanTaskSweeper:
|
|
|
- def __init__(self, redis_conf: Dict):
|
|
|
|
|
- self.redis_client = redis.Redis(**redis_conf)
|
|
|
|
|
|
|
+ def __init__(self, redis_conf: Dict, logger=None):
|
|
|
|
|
+ self.logger = logger or setup_app_logger("Sweeper")
|
|
|
|
|
+ safe_conf = dict(redis_conf)
|
|
|
|
|
+ safe_conf.setdefault('socket_timeout', 5.0)
|
|
|
|
|
+ safe_conf.setdefault('socket_connect_timeout', 5.0)
|
|
|
|
|
+ safe_conf.setdefault('socket_keepalive', True)
|
|
|
|
|
+ safe_conf.setdefault('health_check_interval', 30)
|
|
|
|
|
+ self.redis_client = redis.Redis(**safe_conf)
|
|
|
|
|
+ self._running = True
|
|
|
|
|
+
|
|
|
|
|
+ def stop(self, *args):
|
|
|
|
|
+ if self._running:
|
|
|
|
|
+ self.logger.info("Stopping Task Sweeper gracefully...")
|
|
|
|
|
+ self._running = False
|
|
|
|
|
|
|
|
- def run_forever(self):
|
|
|
|
|
- logging.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
|
|
|
|
|
- while True:
|
|
|
|
|
|
|
+ def run_forever(self, interval: float = 3.0):
|
|
|
|
|
+ self.logger.info("🛡️ Task Sweeper started. Monitoring dead tasks (Heartbeat timeout)...")
|
|
|
|
|
+ while self._running:
|
|
|
try:
|
|
try:
|
|
|
self._sweep()
|
|
self._sweep()
|
|
|
|
|
+ except redis.ConnectionError as ce:
|
|
|
|
|
+ self.logger.warning(f"Redis connection error in sweeper: {ce}. Retrying in 5s...")
|
|
|
|
|
+ time.sleep(5)
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- logging.error(f"Sweeper loop error: {e}")
|
|
|
|
|
|
|
+ self.logger.error(f"Sweeper loop error: {e}")
|
|
|
|
|
|
|
|
- time.sleep(3)
|
|
|
|
|
|
|
+ # 优雅休眠,支持即时响应退出信号
|
|
|
|
|
+ step = 0.5
|
|
|
|
|
+ elapsed = 0.0
|
|
|
|
|
+ while self._running and elapsed < interval:
|
|
|
|
|
+ time.sleep(step)
|
|
|
|
|
+ elapsed += step
|
|
|
|
|
+
|
|
|
|
|
+ self.logger.info("Task Sweeper stopped.")
|
|
|
|
|
|
|
|
def _sweep(self):
|
|
def _sweep(self):
|
|
|
# 扫描集群中所有 Worker 的 tracking keys
|
|
# 扫描集群中所有 Worker 的 tracking keys
|
|
@@ -31,51 +56,73 @@ class OrphanTaskSweeper:
|
|
|
current_time = time.time()
|
|
current_time = time.time()
|
|
|
|
|
|
|
|
for key in tracker_keys:
|
|
for key in tracker_keys:
|
|
|
|
|
+ if not self._running:
|
|
|
|
|
+ break
|
|
|
|
|
+
|
|
|
key_str = key.decode('utf-8') if isinstance(key, bytes) else key
|
|
key_str = key.decode('utf-8') if isinstance(key, bytes) else key
|
|
|
|
|
|
|
|
# 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
|
|
# 【核心逻辑】只要 Deadline (Score) 小于 当前时间,说明心跳彻底断了
|
|
|
dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
|
|
dead_tasks = self.redis_client.zrangebyscore(key_str, 0, current_time)
|
|
|
-
|
|
|
|
|
if not dead_tasks:
|
|
if not dead_tasks:
|
|
|
continue
|
|
continue
|
|
|
|
|
|
|
|
for task_id_bytes in dead_tasks:
|
|
for task_id_bytes in dead_tasks:
|
|
|
- task_id = int(task_id_bytes.decode('utf-8'))
|
|
|
|
|
- self._rescue_task(key_str, task_id)
|
|
|
|
|
|
|
+ if not self._running:
|
|
|
|
|
+ break
|
|
|
|
|
+ try:
|
|
|
|
|
+ task_id = int(task_id_bytes.decode('utf-8') if isinstance(task_id_bytes, bytes) else task_id_bytes)
|
|
|
|
|
+ self._rescue_task(key_str, task_id)
|
|
|
|
|
+ except ValueError:
|
|
|
|
|
+ self.redis_client.zrem(key_str, task_id_bytes)
|
|
|
|
|
|
|
|
def _rescue_task(self, tracker_key: str, task_id: int):
|
|
def _rescue_task(self, tracker_key: str, task_id: int):
|
|
|
- logging.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
|
|
|
|
|
|
|
+ self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
|
|
|
try:
|
|
try:
|
|
|
- task_data = VSCloudApi.Instance().get_vas_task(task_id)
|
|
|
|
|
|
|
+ task_data = None
|
|
|
|
|
+ try:
|
|
|
|
|
+ task_data = VSCloudApi.Instance().get_vas_task(str(task_id))
|
|
|
|
|
+ except BizException as be:
|
|
|
|
|
+ if be.code == 40401 or "not found" in be.message.lower():
|
|
|
|
|
+ self.logger.info(f"Task={task_id} not found in cloud. Removing from local tracker.")
|
|
|
|
|
+ self.redis_client.zrem(tracker_key, task_id)
|
|
|
|
|
+ return
|
|
|
|
|
+ raise be
|
|
|
|
|
|
|
|
if not task_data:
|
|
if not task_data:
|
|
|
- logging.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
|
|
|
|
|
|
|
+ self.logger.info(f"Task={task_id} missing in cloud. Removing from local tracker.")
|
|
|
self.redis_client.zrem(tracker_key, task_id)
|
|
self.redis_client.zrem(tracker_key, task_id)
|
|
|
return
|
|
return
|
|
|
|
|
|
|
|
current_status = task_data.get('status', '')
|
|
current_status = task_data.get('status', '')
|
|
|
-
|
|
|
|
|
- if current_status in ['cancelled', 'pause' ,'grabbed', 'success', 'completed']:
|
|
|
|
|
- logging.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
|
|
|
|
|
|
|
+ if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed']:
|
|
|
|
|
+ self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
|
|
|
self.redis_client.zrem(tracker_key, task_id)
|
|
self.redis_client.zrem(tracker_key, task_id)
|
|
|
return
|
|
return
|
|
|
|
|
|
|
|
# 云端状态还是 processing,证明 Worker 真的死机了,强制归还
|
|
# 云端状态还是 processing,证明 Worker 真的死机了,强制归还
|
|
|
- logging.info(f"Returning dead task={task_id} to queue.")
|
|
|
|
|
|
|
+ self.logger.info(f"Returning dead task={task_id} to queue.")
|
|
|
VSCloudApi.Instance().return_vas_task_to_queue(task_id)
|
|
VSCloudApi.Instance().return_vas_task_to_queue(task_id)
|
|
|
|
|
|
|
|
self.redis_client.zrem(tracker_key, task_id)
|
|
self.redis_client.zrem(tracker_key, task_id)
|
|
|
- logging.info(f"✅ Dead task={task_id} successfully rescued and returned.")
|
|
|
|
|
|
|
+ self.logger.info(f"✅ Dead task={task_id} successfully rescued and returned.")
|
|
|
|
|
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- logging.error(f"Failed to rescue task={task_id}: {e}. Will retry next minute.")
|
|
|
|
|
|
|
+ self.logger.error(f"Failed to rescue task={task_id}: {e}. Will retry next round.")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def main():
|
|
|
|
|
+ logger = setup_app_logger("Sweeper")
|
|
|
|
|
+ sweeper = OrphanTaskSweeper(REDIS_CFG, logger=logger)
|
|
|
|
|
+
|
|
|
|
|
+ def _signal_handler(signum, frame):
|
|
|
|
|
+ logger.info(f"Received signal {signum}. Triggering shutdown...")
|
|
|
|
|
+ sweeper.stop()
|
|
|
|
|
+
|
|
|
|
|
+ signal.signal(signal.SIGINT, _signal_handler)
|
|
|
|
|
+ signal.signal(signal.SIGTERM, _signal_handler)
|
|
|
|
|
+
|
|
|
|
|
+ sweeper.run_forever()
|
|
|
|
|
+
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if __name__ == "__main__":
|
|
|
- config_path = os.path.join(os.path.dirname(__file__), "config", "config.json")
|
|
|
|
|
-
|
|
|
|
|
- with open(config_path, 'r') as f:
|
|
|
|
|
- config_data = json.load(f)
|
|
|
|
|
-
|
|
|
|
|
- redis_conf = config_data.get('redis', {})
|
|
|
|
|
- sweeper = OrphanTaskSweeper(redis_conf)
|
|
|
|
|
- sweeper.run_forever()
|
|
|
|
|
|
|
+ main()
|