| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380 |
- import os
- import time
- import json
- import random
- import threading
- from datetime import datetime
- from typing import List, Dict, Callable, Optional
- from vs_types import (
- GroupConfig, VSPlgConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
- BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
- NetworkTimeoutError
- )
- from vs_plg_factory import VSPlgFactory
- from toolkit.vs_cloud_api import VSCloudApi
- from toolkit.proxy_manager import ProxyManager
- from toolkit.notifier import notify_date_changes
- from utils.safe_redis_cli import SafeRedisClient
- class SentinelGCO:
- def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
- self.m_cfg = cfg
- self.m_factory = VSPlgFactory()
- self.m_logger = logger
- self.m_tasks: List[Task] = []
- self.m_lock = threading.RLock()
- self.m_stop_event = threading.Event()
- self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
- self.m_last_group_query_time = 0.0
- self.m_spawn_cooldown_until = 0.0
- self.m_current_dates = {}
- def _log(self, message):
- if self.m_logger:
- self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
- else:
- print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
-
- def _get_average_interval(self) -> float:
- """计算当前组平均的查询间隔(秒)"""
- mode = self.m_cfg.sentinel.query_wait.mode
- if mode == QueryWaitMode.Loop:
- return 1.0
- elif mode == QueryWaitMode.Fixed:
- return float(self.m_cfg.sentinel.query_wait.fixed_wait)
- elif mode == QueryWaitMode.Random:
- return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
- return 30.0
-
- def update_config(self, new_cfg: GroupConfig):
- """
- 动态更新配置
- """
- with self.m_lock:
- if self.m_cfg.enable and not new_cfg.enable:
- self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
- elif not self.m_cfg.enable and new_cfg.enable:
- self._log("Config dynamically updated: Group ENABLED.")
- else:
- self._log("Config dynamically updated: Parameters refreshed.")
-
- self.m_cfg = new_cfg
- def get_status(self) -> GroupStatus:
- with self.m_lock:
- tasks = list(self.m_tasks)
- gid = self.m_cfg.identifier
- enable = self.m_cfg.enable
- plugin_name = self.m_cfg.plugin_config.plugin_name
- plugins_status = []
- healthy_count = 0
- for t in tasks:
- if t and t.instance:
- st = t.instance.get_plugin_status()
- if st.is_healthy:
- healthy_count += 1
- plugins_status.append(st)
- return GroupStatus(
- group_id=gid,
- enable=enable,
- plugin_name=plugin_name,
- total_tasks=len(tasks),
- healthy_tasks=healthy_count,
- plugins=plugins_status,
- updated_at=utc_now_str(),
- )
- def start(self):
- if not self.m_cfg.enable:
- return
- self._log("Starting Sentinel...")
- plugin_name = self.m_cfg.plugin_config.plugin_name
- class_name = "".join(part.title() for part in plugin_name.split('_'))
- plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
- self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
- threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
- threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
- def stop(self):
- self._log("Stopping Sentinel...")
- self.m_stop_event.set()
- with self.m_lock:
- tasks_to_cleanup = list(self.m_tasks)
- self.m_tasks.clear()
- for task in tasks_to_cleanup:
- self._cleanup_task(task, "sentinel stopped")
- def _cleanup_instance(self, instance, reason: str = ""):
- try:
- if instance:
- self._log(f"🧹 Cleaned up instance. Reason: {reason}")
- instance.cleanup()
- except Exception as e:
- self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
- def _cleanup_task(self, task: Task, reason: str):
- if task and task.instance:
- self._cleanup_instance(task.instance, reason)
- def _remove_task(self, task: Task, reason: str):
- removed = False
- with self.m_lock:
- if task in self.m_tasks:
- self.m_tasks.remove(task)
- removed = True
- if removed:
- self._cleanup_task(task, reason)
- def _get_redis_key(self, routing_key: str) -> str:
- return f"vs:signal:{routing_key}"
- def _normalize_error(self, err: Exception) -> BizException:
- """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
- if isinstance(err, BizException):
- return err
- err_str = str(err)
- if "42901" in err_str or "Rate limited" in err_str:
- return RateLimiteddError(message=err_str)
- if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
- return SessionExpiredOrInvalidError(message=err_str)
- if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str:
- return PermissionDeniedError(message=err_str)
- return BizException(
- code=50000,
- message=f"Unhandled ({type(err).__name__}): {err_str}",
- )
- def _dispatch_error_policy(
- self,
- action: str,
- err: Exception,
- task: Optional[Task] = None,
- plg_cfg: Optional[VSPlgConfig] = None
- ) -> None:
- """
- Sentinel 错误调度与自愈策略引擎 (Policy Engine):
- 基于 (Action, 异常事实类型) 矩阵 + 哨兵池模式闭环执行自愈动作:
- - spawn/login 时 429:判定为账号级限流,锁定账号并设置 m_spawn_cooldown_until 退避;
- - query 时 429:会话已登录,判定为出口 IP 频率超限,不锁定账号,移除任务以触发 creator 换代理生成新实例;
- - session expired / 401:移除故障任务;
- - 403 Forbidden / WAF:判定为出口 IP 被风控,移除任务以轮换代理;
- - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
- """
- biz_err = self._normalize_error(err)
- acc = None
- if plg_cfg and plg_cfg.account:
- acc = plg_cfg.account
- elif task and getattr(task.instance, "config", None) and task.instance.config.account:
- acc = task.instance.config.account
- # 1. 目标站限流 / 429
- if isinstance(biz_err, RateLimiteddError):
- if action in ("spawn", "login", "create_session"):
- backoff = float(self.m_cfg.login_backoff)
- self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
- if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0):
- try:
- VSCloudApi.Instance().lock_account(acc.id, int(backoff))
- self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
- except Exception as lock_err:
- self._log(f"Failed to lock account via cloud API: {lock_err}")
- self.m_spawn_cooldown_until = time.time() + backoff
- if task:
- self._remove_task(task, "spawn account rate limited")
- elif action == "query":
- # 查询时 429:出口 IP 限流,绝不锁账号,移除当前 task 触发自动换新代理!
- self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
- if task:
- self._remove_task(task, "query ip rate limited")
- else:
- self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
- if task:
- task.next_run = time.time() + 10.0
- # 2. 会话过期 / 401
- elif isinstance(biz_err, SessionExpiredOrInvalidError):
- if action in ("spawn", "login", "create_session"):
- self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
- self.m_spawn_cooldown_until = time.time() + 5.0
- else:
- self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
- if task:
- self._remove_task(task, f"{action} session expired")
- # 3. 权限被拒 / WAF 拦截 / 403
- elif isinstance(biz_err, PermissionDeniedError):
- proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)
- self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
- if task:
- self._remove_task(task, f"{action} waf blocked")
- if action == "spawn":
- self.m_spawn_cooldown_until = time.time() + proxy_cd
- # 4. 网络或浏览器超时
- elif isinstance(biz_err, NetworkTimeoutError):
- self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
- if task:
- task.next_run = time.time() + 5.0
- # 5. 兜底策略
- else:
- self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
- if action == "spawn":
- self.m_spawn_cooldown_until = time.time() + 10.0
- elif task:
- task.next_run = time.time() + 10.0
- def _is_within_active_hours(self) -> bool:
- """
- 判断当前是否在允许创建实例的时间段内
- """
- start_str = self.m_cfg.active_time_start
- end_str = self.m_cfg.active_time_end
- current_bj_time = datetime.utcnow().time()
- start_time = datetime.strptime(start_str, "%H:%M").time()
- end_time = datetime.strptime(end_str, "%H:%M").time()
- return start_time <= current_bj_time <= end_time
- def _monitor_loop(self):
- self._log("Monitor loop started.")
-
- self.m_last_group_query_time = 0.0
- while not self.m_stop_event.is_set():
- try:
- time.sleep(1)
- now = time.time()
-
- with self.m_lock:
- tasks_to_check = list(self.m_tasks)
- active_tasks = []
- dead_tasks = []
- for t in tasks_to_check:
- if t.instance.health_check():
- active_tasks.append(t)
- else:
- dead_tasks.append(t)
-
- if dead_tasks:
- with self.m_lock:
- current_tasks = list(self.m_tasks)
- self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
- for t in dead_tasks:
- if t in current_tasks:
- self._cleanup_task(t, "health check failed")
- else:
- with self.m_lock:
- self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
-
- if not active_tasks:
- continue
- avg_interval = self._get_average_interval()
- global_gap = max(1.0, avg_interval / len(active_tasks))
-
- active_tasks.sort(key=lambda x: x.next_run)
- for task in active_tasks:
- if now < task.next_run:
- continue
-
- if now - self.m_last_group_query_time < global_gap:
- break
- apt_types = self.m_cfg.appointment_types
- if not apt_types:
- continue
- weights = [max(0.0, float(item.weight)) for item in apt_types]
- if sum(weights) > 0:
- apt_type = random.choices(apt_types, weights=weights, k=1)[0]
- else:
- apt_type = random.choice(apt_types)
-
- interval = 30
- mode = task.qw_cfg.mode
- if mode == QueryWaitMode.Loop:
- interval = 1
- elif mode == QueryWaitMode.Fixed:
- interval = task.qw_cfg.fixed_wait
- elif mode == QueryWaitMode.Random:
- interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
- self.m_last_group_query_time = now
- try:
- result = task.instance.query(apt_type)
- result.apt_type = apt_type
- notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
- if result.success:
- ttl = self.m_cfg.sentinel.signal_ttl
- self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
- payload = {
- "group_id": self.m_cfg.identifier,
- "apt_type": apt_type.model_dump(),
- "query_result": result.to_snapshot_payload(),
- "timestamp": time.time()
- }
- redis_key = self._get_redis_key(apt_type.routing_key)
- self.redis_client.setex(redis_key, ttl, json.dumps(payload))
- task.next_run = time.time() + interval
- except Exception as e:
- self._dispatch_error_policy("query", e, task=task)
- break
- except Exception as e:
- self._dispatch_error_policy("monitor_loop", e)
- def _creator_loop(self):
- self._log("Creator loop started.")
- while not self.m_stop_event.wait(1.0):
- if not self._is_within_active_hours():
- continue
- if time.time() < self.m_spawn_cooldown_until:
- continue
-
- with self.m_lock:
- current = len(self.m_tasks)
- target = self.m_cfg.sentinel.target_instances
-
- if current < target:
- self._spawn_sentinel_worker()
- def _spawn_sentinel_worker(self):
- instance = None
- task = None
- success = False
- plg_cfg = None
- try:
- plg_cfg = VSPlgConfig()
- plg_cfg.debug = self.m_cfg.debug
- plg_cfg.free_config = self.m_cfg.free_config
- plg_cfg.session_max_life = self.m_cfg.session_max_life
-
- if not self.m_cfg.need_account:
- plg_cfg.account.id = 0
- plg_cfg.account.username = "Guest"
- else:
- acc = VSCloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
- plg_cfg.account = type(plg_cfg.account)(**acc)
-
- if self.m_cfg.need_proxy:
- proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
- plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
- instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
- instance.set_log(self.m_logger)
- instance.set_config(plg_cfg)
-
- instance.create_session()
-
- task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
- with self.m_lock:
- self.m_tasks.append(task)
- success = True
- self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
- except Exception as e:
- self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
- finally:
- if not success and instance:
- self._cleanup_instance(instance, "spawn failed")
|