sentinel.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from vs_types import (
  9. GroupConfig, VSPlgConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
  10. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  11. NetworkTimeoutError
  12. )
  13. from vs_plg_factory import VSPlgFactory
  14. from toolkit.vs_cloud_api import VSCloudApi
  15. from toolkit.proxy_manager import ProxyManager
  16. from toolkit.notifier import notify_date_changes
  17. from utils.safe_redis_cli import SafeRedisClient
  18. class SentinelGCO:
  19. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  20. self.m_cfg = cfg
  21. self.m_factory = VSPlgFactory()
  22. self.m_logger = logger
  23. self.m_tasks: List[Task] = []
  24. self.m_lock = threading.RLock()
  25. self.m_stop_event = threading.Event()
  26. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  27. self.m_last_group_query_time = 0.0
  28. self.m_spawn_cooldown_until = 0.0
  29. self.m_current_dates = {}
  30. def _log(self, message):
  31. if self.m_logger:
  32. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  33. else:
  34. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  35. def _get_average_interval(self) -> float:
  36. """计算当前组平均的查询间隔(秒)"""
  37. mode = self.m_cfg.sentinel.query_wait.mode
  38. if mode == QueryWaitMode.Loop:
  39. return 1.0
  40. elif mode == QueryWaitMode.Fixed:
  41. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  42. elif mode == QueryWaitMode.Random:
  43. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  44. return 30.0
  45. def update_config(self, new_cfg: GroupConfig):
  46. """
  47. 动态更新配置
  48. """
  49. with self.m_lock:
  50. if self.m_cfg.enable and not new_cfg.enable:
  51. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  52. elif not self.m_cfg.enable and new_cfg.enable:
  53. self._log("Config dynamically updated: Group ENABLED.")
  54. else:
  55. self._log("Config dynamically updated: Parameters refreshed.")
  56. self.m_cfg = new_cfg
  57. def get_status(self) -> GroupStatus:
  58. with self.m_lock:
  59. tasks = list(self.m_tasks)
  60. gid = self.m_cfg.identifier
  61. enable = self.m_cfg.enable
  62. plugin_name = self.m_cfg.plugin_config.plugin_name
  63. plugins_status = []
  64. healthy_count = 0
  65. for t in tasks:
  66. if t and t.instance:
  67. st = t.instance.get_plugin_status()
  68. if st.is_healthy:
  69. healthy_count += 1
  70. plugins_status.append(st)
  71. return GroupStatus(
  72. group_id=gid,
  73. enable=enable,
  74. plugin_name=plugin_name,
  75. total_tasks=len(tasks),
  76. healthy_tasks=healthy_count,
  77. plugins=plugins_status,
  78. updated_at=utc_now_str(),
  79. )
  80. def start(self):
  81. if not self.m_cfg.enable:
  82. return
  83. self._log("Starting Sentinel...")
  84. plugin_name = self.m_cfg.plugin_config.plugin_name
  85. class_name = "".join(part.title() for part in plugin_name.split('_'))
  86. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  87. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  88. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  89. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  90. def stop(self):
  91. self._log("Stopping Sentinel...")
  92. self.m_stop_event.set()
  93. with self.m_lock:
  94. tasks_to_cleanup = list(self.m_tasks)
  95. self.m_tasks.clear()
  96. for task in tasks_to_cleanup:
  97. self._cleanup_task(task, "sentinel stopped")
  98. def _cleanup_instance(self, instance, reason: str = ""):
  99. try:
  100. if instance:
  101. self._log(f"🧹 Cleaned up instance. Reason: {reason}")
  102. instance.cleanup()
  103. except Exception as e:
  104. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  105. def _cleanup_task(self, task: Task, reason: str):
  106. if task and task.instance:
  107. self._cleanup_instance(task.instance, reason)
  108. def _remove_task(self, task: Task, reason: str):
  109. removed = False
  110. with self.m_lock:
  111. if task in self.m_tasks:
  112. self.m_tasks.remove(task)
  113. removed = True
  114. if removed:
  115. self._cleanup_task(task, reason)
  116. def _get_redis_key(self, routing_key: str) -> str:
  117. return f"vs:signal:{routing_key}"
  118. def _normalize_error(self, err: Exception) -> BizException:
  119. """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
  120. if isinstance(err, BizException):
  121. return err
  122. err_str = str(err)
  123. if "42901" in err_str or "Rate limited" in err_str:
  124. return RateLimiteddError(message=err_str)
  125. if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
  126. return SessionExpiredOrInvalidError(message=err_str)
  127. if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str:
  128. return PermissionDeniedError(message=err_str)
  129. return BizException(
  130. code=50000,
  131. message=f"Unhandled ({type(err).__name__}): {err_str}",
  132. )
  133. def _dispatch_error_policy(
  134. self,
  135. action: str,
  136. err: Exception,
  137. task: Optional[Task] = None,
  138. plg_cfg: Optional[VSPlgConfig] = None
  139. ) -> None:
  140. """
  141. Sentinel 错误调度与自愈策略引擎 (Policy Engine):
  142. 基于 (Action, 异常事实类型) 矩阵 + 哨兵池模式闭环执行自愈动作:
  143. - spawn/login 时 429:判定为账号级限流,锁定账号并设置 m_spawn_cooldown_until 退避;
  144. - query 时 429:会话已登录,判定为出口 IP 频率超限,不锁定账号,移除任务以触发 creator 换代理生成新实例;
  145. - session expired / 401:移除故障任务;
  146. - 403 Forbidden / WAF:判定为出口 IP 被风控,移除任务以轮换代理;
  147. - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
  148. """
  149. biz_err = self._normalize_error(err)
  150. acc = None
  151. if plg_cfg and plg_cfg.account:
  152. acc = plg_cfg.account
  153. elif task and getattr(task.instance, "config", None) and task.instance.config.account:
  154. acc = task.instance.config.account
  155. # 1. 目标站限流 / 429
  156. if isinstance(biz_err, RateLimiteddError):
  157. if action in ("spawn", "login", "create_session"):
  158. backoff = float(self.m_cfg.login_backoff)
  159. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  160. if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0):
  161. try:
  162. VSCloudApi.Instance().lock_account(acc.id, int(backoff))
  163. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  164. except Exception as lock_err:
  165. self._log(f"Failed to lock account via cloud API: {lock_err}")
  166. self.m_spawn_cooldown_until = time.time() + backoff
  167. if task:
  168. self._remove_task(task, "spawn account rate limited")
  169. elif action == "query":
  170. # 查询时 429:出口 IP 限流,绝不锁账号,移除当前 task 触发自动换新代理!
  171. self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
  172. if task:
  173. self._remove_task(task, "query ip rate limited")
  174. else:
  175. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  176. if task:
  177. task.next_run = time.time() + 10.0
  178. # 2. 会话过期 / 401
  179. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  180. if action in ("spawn", "login", "create_session"):
  181. self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
  182. self.m_spawn_cooldown_until = time.time() + 5.0
  183. else:
  184. self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
  185. if task:
  186. self._remove_task(task, f"{action} session expired")
  187. # 3. 权限被拒 / WAF 拦截 / 403
  188. elif isinstance(biz_err, PermissionDeniedError):
  189. proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)
  190. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  191. if task:
  192. self._remove_task(task, f"{action} waf blocked")
  193. if action == "spawn":
  194. self.m_spawn_cooldown_until = time.time() + proxy_cd
  195. # 4. 网络或浏览器超时
  196. elif isinstance(biz_err, NetworkTimeoutError):
  197. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  198. if task:
  199. task.next_run = time.time() + 5.0
  200. # 5. 兜底策略
  201. else:
  202. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  203. if action == "spawn":
  204. self.m_spawn_cooldown_until = time.time() + 10.0
  205. elif task:
  206. task.next_run = time.time() + 10.0
  207. def _is_within_active_hours(self) -> bool:
  208. """
  209. 判断当前是否在允许创建实例的时间段内
  210. """
  211. start_str = self.m_cfg.active_time_start
  212. end_str = self.m_cfg.active_time_end
  213. current_bj_time = datetime.utcnow().time()
  214. start_time = datetime.strptime(start_str, "%H:%M").time()
  215. end_time = datetime.strptime(end_str, "%H:%M").time()
  216. return start_time <= current_bj_time <= end_time
  217. def _monitor_loop(self):
  218. self._log("Monitor loop started.")
  219. self.m_last_group_query_time = 0.0
  220. while not self.m_stop_event.is_set():
  221. try:
  222. time.sleep(1)
  223. now = time.time()
  224. with self.m_lock:
  225. tasks_to_check = list(self.m_tasks)
  226. active_tasks = []
  227. dead_tasks = []
  228. for t in tasks_to_check:
  229. if t.instance.health_check():
  230. active_tasks.append(t)
  231. else:
  232. dead_tasks.append(t)
  233. if dead_tasks:
  234. with self.m_lock:
  235. current_tasks = list(self.m_tasks)
  236. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  237. for t in dead_tasks:
  238. if t in current_tasks:
  239. self._cleanup_task(t, "health check failed")
  240. else:
  241. with self.m_lock:
  242. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  243. if not active_tasks:
  244. continue
  245. avg_interval = self._get_average_interval()
  246. global_gap = max(1.0, avg_interval / len(active_tasks))
  247. active_tasks.sort(key=lambda x: x.next_run)
  248. for task in active_tasks:
  249. if now < task.next_run:
  250. continue
  251. if now - self.m_last_group_query_time < global_gap:
  252. break
  253. apt_types = self.m_cfg.appointment_types
  254. if not apt_types:
  255. continue
  256. weights = [max(0.0, float(item.weight)) for item in apt_types]
  257. if sum(weights) > 0:
  258. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  259. else:
  260. apt_type = random.choice(apt_types)
  261. interval = 30
  262. mode = task.qw_cfg.mode
  263. if mode == QueryWaitMode.Loop:
  264. interval = 1
  265. elif mode == QueryWaitMode.Fixed:
  266. interval = task.qw_cfg.fixed_wait
  267. elif mode == QueryWaitMode.Random:
  268. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  269. self.m_last_group_query_time = now
  270. try:
  271. result = task.instance.query(apt_type)
  272. result.apt_type = apt_type
  273. notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
  274. if result.success:
  275. ttl = self.m_cfg.sentinel.signal_ttl
  276. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  277. payload = {
  278. "group_id": self.m_cfg.identifier,
  279. "apt_type": apt_type.model_dump(),
  280. "query_result": result.to_snapshot_payload(),
  281. "timestamp": time.time()
  282. }
  283. redis_key = self._get_redis_key(apt_type.routing_key)
  284. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  285. task.next_run = time.time() + interval
  286. except Exception as e:
  287. self._dispatch_error_policy("query", e, task=task)
  288. break
  289. except Exception as e:
  290. self._dispatch_error_policy("monitor_loop", e)
  291. def _creator_loop(self):
  292. self._log("Creator loop started.")
  293. while not self.m_stop_event.wait(1.0):
  294. if not self._is_within_active_hours():
  295. continue
  296. if time.time() < self.m_spawn_cooldown_until:
  297. continue
  298. with self.m_lock:
  299. current = len(self.m_tasks)
  300. target = self.m_cfg.sentinel.target_instances
  301. if current < target:
  302. self._spawn_sentinel_worker()
  303. def _spawn_sentinel_worker(self):
  304. instance = None
  305. task = None
  306. success = False
  307. plg_cfg = None
  308. try:
  309. plg_cfg = VSPlgConfig()
  310. plg_cfg.debug = self.m_cfg.debug
  311. plg_cfg.free_config = self.m_cfg.free_config
  312. plg_cfg.session_max_life = self.m_cfg.session_max_life
  313. if not self.m_cfg.need_account:
  314. plg_cfg.account.id = 0
  315. plg_cfg.account.username = "Guest"
  316. else:
  317. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  318. plg_cfg.account = type(plg_cfg.account)(**acc)
  319. if self.m_cfg.need_proxy:
  320. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  321. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  322. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  323. instance.set_log(self.m_logger)
  324. instance.set_config(plg_cfg)
  325. instance.create_session()
  326. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  327. with self.m_lock:
  328. self.m_tasks.append(task)
  329. success = True
  330. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  331. except Exception as e:
  332. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  333. finally:
  334. if not success and instance:
  335. self._cleanup_instance(instance, "spawn failed")