import os import time import json import random import threading from datetime import datetime from typing import List, Dict, Callable, Optional from core_types import ( GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str, BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError, NetworkTimeoutError ) from plugin_factory import PluginFactory from toolkit.cloud_api import CloudApi from toolkit.proxy_manager import ProxyManager from toolkit.notifier import notify_date_changes from utils.safe_redis_cli import SafeRedisClient class SentinelGCO: def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): self.m_cfg = cfg self.m_factory = PluginFactory() self.m_logger = logger self.m_tasks: List[Task] = [] self.m_lock = threading.RLock() self.m_stop_event = threading.Event() self.redis_client = SafeRedisClient(redis_conf, self.m_logger) self.m_last_group_query_time = 0.0 self.m_spawn_cooldown_until = 0.0 self.m_current_dates = {} def _log(self, message): if self.m_logger: self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}') else: print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}') def _get_average_interval(self) -> float: """计算当前组平均的查询间隔(秒)""" mode = self.m_cfg.sentinel.query_wait.mode if mode == QueryWaitMode.Loop: return 1.0 elif mode == QueryWaitMode.Fixed: return float(self.m_cfg.sentinel.query_wait.fixed_wait) elif mode == QueryWaitMode.Random: return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0 return 30.0 def update_config(self, new_cfg: GroupConfig): """ 动态更新配置 """ with self.m_lock: if self.m_cfg.enable and not new_cfg.enable: self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.") elif not self.m_cfg.enable and new_cfg.enable: self._log("Config dynamically updated: Group ENABLED.") else: self._log("Config dynamically updated: Parameters refreshed.") self.m_cfg = new_cfg def get_status(self) -> GroupStatus: with self.m_lock: tasks = list(self.m_tasks) gid = self.m_cfg.identifier enable = self.m_cfg.enable plugin_name = self.m_cfg.plugin_config.plugin_name plugins_status = [] healthy_count = 0 for t in tasks: if t and t.instance: st = t.instance.get_plugin_status() if st.is_healthy: healthy_count += 1 plugins_status.append(st) return GroupStatus( group_id=gid, enable=enable, plugin_name=plugin_name, total_tasks=len(tasks), healthy_tasks=healthy_count, plugins=plugins_status, updated_at=utc_now_str(), ) def start(self): if not self.m_cfg.enable: return self._log("Starting Sentinel...") plugin_name = self.m_cfg.plugin_config.plugin_name class_name = "".join(part.title() for part in plugin_name.split('_')) plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin) self.m_factory.register_plugin(plugin_name, plugin_path, class_name) threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start() threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start() def stop(self): self._log("Stopping Sentinel...") self.m_stop_event.set() with self.m_lock: tasks_to_cleanup = list(self.m_tasks) self.m_tasks.clear() for task in tasks_to_cleanup: self._cleanup_task(task, "sentinel stopped") def _cleanup_instance(self, instance, reason: str = ""): try: if instance: self._log(f"🧹 Cleaned up instance. Reason: {reason}") instance.cleanup() except Exception as e: self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}") def _cleanup_task(self, task: Task, reason: str): if task and task.instance: self._cleanup_instance(task.instance, reason) def _remove_task(self, task: Task, reason: str): removed = False with self.m_lock: if task in self.m_tasks: self.m_tasks.remove(task) removed = True if removed: self._cleanup_task(task, reason) def _get_redis_key(self, routing_key: str) -> str: return f"vs:signal:{routing_key}" def _normalize_error(self, err: Exception) -> BizException: """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容""" err_str = str(err) # 如果是底层网络/CORS预检被拒导致的 Fetch 失败或 WAF 阻断,精准提升为 PermissionDeniedError if "Failed to fetch" in err_str or "Status: 0" in err_str or "WAF Blocked" in err_str or "cf-mitigated" in err_str: return PermissionDeniedError(message=err_str) if isinstance(err, BizException): return err if "42901" in err_str or "Rate limited" in err_str: return RateLimiteddError(message=err_str) if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower(): return SessionExpiredOrInvalidError(message=err_str) if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str: return PermissionDeniedError(message=err_str) return BizException( code=50000, message=f"Unhandled ({type(err).__name__}): {err_str}", ) def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None: """安全归还账号:将云端冷却时间重置为 0,避免非账号问题导致账号被误锁 3 小时""" if acc and acc.id > 0 and acc.username not in ("", "Guest"): try: CloudApi.Instance().lock_account(acc.id, 0) self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).") except Exception as e: self._log(f"Failed to release account [{acc.username}]: {e}") def _dispatch_error_policy( self, action: str, err: Exception, task: Optional[Task] = None, plg_cfg: Optional[PluginConfig] = None ) -> None: """ Sentinel 错误调度与自愈策略引擎 (Policy Engine): 基于 (Action, 异常事实类型) 矩阵 + 哨兵池模式闭环执行自愈动作: - spawn/login 时 429:判定为账号级限流,锁定账号并设置 m_spawn_cooldown_until 退避; - query 时 429:会话已登录,判定为出口 IP 频率超限,不锁定账号,移除任务以触发 creator 换代理生成新实例; - session expired / 401:移除故障任务; - 403 Forbidden / WAF:判定为出口 IP 被风控,移除任务以轮换代理;在 spawn 时立即安全释放账号; - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。 """ biz_err = self._normalize_error(err) active_plg_cfg = plg_cfg or (task.instance.config if task else None) acc = active_plg_cfg.account if active_plg_cfg else None # 1. 目标站限流 / 429 if isinstance(biz_err, RateLimiteddError): if action in ("spawn", "login", "create_session"): backoff = float(self.m_cfg.login_backoff) self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).") if acc and acc.username not in ("", "Guest") and acc.id > 0: try: CloudApi.Instance().lock_account(acc.id, int(backoff)) self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.") except Exception as lock_err: self._log(f"Failed to lock account via cloud API: {lock_err}") self.m_spawn_cooldown_until = time.time() + backoff if task: self._remove_task(task, "spawn account rate limited") elif action == "query": # 查询时 429:出口 IP 限流,绝不锁账号,移除当前 task 触发自动换新代理! self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...") if task: self._remove_task(task, "query ip rate limited") else: self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.") if task: task.next_run = time.time() + 10.0 # 2. 会话过期 / 401 elif isinstance(biz_err, SessionExpiredOrInvalidError): if action in ("spawn", "login", "create_session"): self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...") self.m_spawn_cooldown_until = time.time() + 5.0 else: self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...") if task: self._remove_task(task, f"{action} session expired") # 3. 权限被拒 / WAF 拦截 / 403 elif isinstance(biz_err, PermissionDeniedError): proxy_cd = float(self.m_cfg.proxy_cd or 60.0) self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...") if task: self._remove_task(task, f"{action} waf blocked") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="WAF Blocked") self.m_spawn_cooldown_until = time.time() + proxy_cd # 4. 网络或浏览器超时 elif isinstance(biz_err, NetworkTimeoutError): self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.") if task: task.next_run = time.time() + 5.0 if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="Network Timeout") self.m_spawn_cooldown_until = time.time() + 10.0 # 5. 兜底策略 else: self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}") self.m_spawn_cooldown_until = time.time() + 10.0 elif task: task.next_run = time.time() + 10.0 # 6. 统一代理健康指标埋点 if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0: err_type = biz_err.message if isinstance(biz_err, PermissionDeniedError): if "403202" in err_type or "cf-mitigated" in err_type.lower() or "waf blocked" in err_type.lower(): err_type = "cf-mitigated (403202)" ProxyManager.Instance().mark_proxy_result( proxy=active_plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action=action, success=False, http_code=biz_err.http_status, error_type=err_type ) def _is_within_active_hours(self) -> bool: """ 判断当前是否在允许创建实例的时间段内 """ start_str = self.m_cfg.active_time_start end_str = self.m_cfg.active_time_end current_bj_time = datetime.utcnow().time() start_time = datetime.strptime(start_str, "%H:%M").time() end_time = datetime.strptime(end_str, "%H:%M").time() return start_time <= current_bj_time <= end_time def _monitor_loop(self): self._log("Monitor loop started.") self.m_last_group_query_time = 0.0 while not self.m_stop_event.is_set(): try: time.sleep(1) now = time.time() with self.m_lock: tasks_to_check = list(self.m_tasks) active_tasks = [] dead_tasks = [] for t in tasks_to_check: if t.instance.health_check(): active_tasks.append(t) else: dead_tasks.append(t) if dead_tasks: with self.m_lock: current_tasks = list(self.m_tasks) self.m_tasks = [t for t in self.m_tasks if t in active_tasks] for t in dead_tasks: if t in current_tasks: self._cleanup_task(t, "health check failed") else: with self.m_lock: self.m_tasks = [t for t in self.m_tasks if t in active_tasks] if not active_tasks: continue avg_interval = self._get_average_interval() global_gap = max(1.0, avg_interval / len(active_tasks)) active_tasks.sort(key=lambda x: x.next_run) for task in active_tasks: if now < task.next_run: continue if now - self.m_last_group_query_time < global_gap: break apt_types = self.m_cfg.appointment_types if not apt_types: continue weights = [max(0.0, float(item.weight)) for item in apt_types] if sum(weights) > 0: apt_type = random.choices(apt_types, weights=weights, k=1)[0] else: apt_type = random.choice(apt_types) interval = 30 mode = task.qw_cfg.mode if mode == QueryWaitMode.Loop: interval = 1 elif mode == QueryWaitMode.Fixed: interval = task.qw_cfg.fixed_wait elif mode == QueryWaitMode.Random: interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max) self.m_last_group_query_time = now try: result = task.instance.query(apt_type) result.apt_type = apt_type notify_date_changes(result, cache=self.m_current_dates, logger=self._log) if result.success: ttl = self.m_cfg.sentinel.signal_ttl self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)") payload = { "group_id": self.m_cfg.identifier, "apt_type": apt_type.model_dump(), "query_result": result.to_snapshot_payload(), "timestamp": time.time() } redis_key = self._get_redis_key(apt_type.routing_key) self.redis_client.setex(redis_key, ttl, json.dumps(payload)) task.next_run = time.time() + interval # 埋点上报 query 成功 if task.instance.config.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=task.instance.config.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="query", success=True, http_code=200 ) except Exception as e: self._dispatch_error_policy("query", e, task=task) break except Exception as e: self._dispatch_error_policy("monitor_loop", e) def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue with self.m_lock: current = len(self.m_tasks) target = self.m_cfg.sentinel.target_instances if current < target: self._spawn_sentinel_worker() def _spawn_sentinel_worker(self): instance = None task = None success = False plg_cfg = None try: plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life if not self.m_cfg.need_account: plg_cfg.account.id = 0 plg_cfg.account.username = "Guest" else: acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd) plg_cfg.account = type(plg_cfg.account)(**acc) if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy( self.m_cfg.proxy_pool, self.m_cfg.proxy_cd, plugin_name=self.m_cfg.plugin_config.plugin_name ) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False) with self.m_lock: self.m_tasks.append(task) success = True if plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="spawn", success=True, http_code=200 ) self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed")