sentinel.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from core_types import (
  9. GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
  10. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  11. NetworkTimeoutError
  12. )
  13. from plugin_factory import PluginFactory
  14. from toolkit.cloud_api import CloudApi
  15. from toolkit.proxy_manager import ProxyManager
  16. from toolkit.notifier import notify_date_changes
  17. from utils.safe_redis_cli import SafeRedisClient
  18. class SentinelGCO:
  19. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  20. self.m_cfg = cfg
  21. self.m_factory = PluginFactory()
  22. self.m_logger = logger
  23. self.m_tasks: List[Task] = []
  24. self.m_lock = threading.RLock()
  25. self.m_stop_event = threading.Event()
  26. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  27. self.m_last_group_query_time = 0.0
  28. self.m_spawn_cooldown_until = 0.0
  29. self.m_current_dates = {}
  30. def _log(self, message):
  31. if self.m_logger:
  32. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  33. else:
  34. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  35. def _get_average_interval(self) -> float:
  36. """计算当前组平均的查询间隔(秒)"""
  37. mode = self.m_cfg.sentinel.query_wait.mode
  38. if mode == QueryWaitMode.Loop:
  39. return 1.0
  40. elif mode == QueryWaitMode.Fixed:
  41. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  42. elif mode == QueryWaitMode.Random:
  43. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  44. return 30.0
  45. def update_config(self, new_cfg: GroupConfig):
  46. """
  47. 动态更新配置
  48. """
  49. with self.m_lock:
  50. if self.m_cfg.enable and not new_cfg.enable:
  51. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  52. elif not self.m_cfg.enable and new_cfg.enable:
  53. self._log("Config dynamically updated: Group ENABLED.")
  54. else:
  55. self._log("Config dynamically updated: Parameters refreshed.")
  56. self.m_cfg = new_cfg
  57. def get_status(self) -> GroupStatus:
  58. with self.m_lock:
  59. tasks = list(self.m_tasks)
  60. gid = self.m_cfg.identifier
  61. enable = self.m_cfg.enable
  62. plugin_name = self.m_cfg.plugin_config.plugin_name
  63. plugins_status = []
  64. healthy_count = 0
  65. for t in tasks:
  66. if t and t.instance:
  67. st = t.instance.get_plugin_status()
  68. if st.is_healthy:
  69. healthy_count += 1
  70. plugins_status.append(st)
  71. return GroupStatus(
  72. group_id=gid,
  73. enable=enable,
  74. plugin_name=plugin_name,
  75. total_tasks=len(tasks),
  76. healthy_tasks=healthy_count,
  77. plugins=plugins_status,
  78. updated_at=utc_now_str(),
  79. )
  80. def start(self):
  81. if not self.m_cfg.enable:
  82. return
  83. self._log("Starting Sentinel...")
  84. plugin_name = self.m_cfg.plugin_config.plugin_name
  85. class_name = "".join(part.title() for part in plugin_name.split('_'))
  86. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  87. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  88. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  89. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  90. def stop(self):
  91. self._log("Stopping Sentinel...")
  92. self.m_stop_event.set()
  93. with self.m_lock:
  94. tasks_to_cleanup = list(self.m_tasks)
  95. self.m_tasks.clear()
  96. for task in tasks_to_cleanup:
  97. self._cleanup_task(task, "sentinel stopped")
  98. def _cleanup_instance(self, instance, reason: str = ""):
  99. try:
  100. if instance:
  101. self._log(f"🧹 Cleaned up instance. Reason: {reason}")
  102. instance.cleanup()
  103. except Exception as e:
  104. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  105. def _cleanup_task(self, task: Task, reason: str):
  106. if task and task.instance:
  107. self._cleanup_instance(task.instance, reason)
  108. def _remove_task(self, task: Task, reason: str):
  109. removed = False
  110. with self.m_lock:
  111. if task in self.m_tasks:
  112. self.m_tasks.remove(task)
  113. removed = True
  114. if removed:
  115. self._cleanup_task(task, reason)
  116. def _get_redis_key(self, routing_key: str) -> str:
  117. return f"vs:signal:{routing_key}"
  118. def _normalize_error(self, err: Exception) -> BizException:
  119. """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
  120. err_str = str(err)
  121. # 如果是底层网络/CORS预检被拒导致的 Fetch 失败或 WAF 阻断,精准提升为 PermissionDeniedError
  122. if "Failed to fetch" in err_str or "Status: 0" in err_str or "WAF Blocked" in err_str or "cf-mitigated" in err_str:
  123. return PermissionDeniedError(message=err_str)
  124. if isinstance(err, BizException):
  125. return err
  126. if "42901" in err_str or "Rate limited" in err_str:
  127. return RateLimiteddError(message=err_str)
  128. if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
  129. return SessionExpiredOrInvalidError(message=err_str)
  130. if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str:
  131. return PermissionDeniedError(message=err_str)
  132. return BizException(
  133. code=50000,
  134. message=f"Unhandled ({type(err).__name__}): {err_str}",
  135. )
  136. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  137. """安全归还账号:将云端冷却时间重置为 0,避免非账号问题导致账号被误锁 3 小时"""
  138. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  139. try:
  140. CloudApi.Instance().lock_account(acc.id, 0)
  141. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  142. except Exception as e:
  143. self._log(f"Failed to release account [{acc.username}]: {e}")
  144. def _dispatch_error_policy(
  145. self,
  146. action: str,
  147. err: Exception,
  148. task: Optional[Task] = None,
  149. plg_cfg: Optional[PluginConfig] = None
  150. ) -> None:
  151. """
  152. Sentinel 错误调度与自愈策略引擎 (Policy Engine):
  153. 基于 (Action, 异常事实类型) 矩阵 + 哨兵池模式闭环执行自愈动作:
  154. - spawn/login 时 429:判定为账号级限流,锁定账号并设置 m_spawn_cooldown_until 退避;
  155. - query 时 429:会话已登录,判定为出口 IP 频率超限,不锁定账号,移除任务以触发 creator 换代理生成新实例;
  156. - session expired / 401:移除故障任务;
  157. - 403 Forbidden / WAF:判定为出口 IP 被风控,移除任务以轮换代理;在 spawn 时立即安全释放账号;
  158. - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
  159. """
  160. biz_err = self._normalize_error(err)
  161. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  162. acc = active_plg_cfg.account if active_plg_cfg else None
  163. # 1. 目标站限流 / 429
  164. if isinstance(biz_err, RateLimiteddError):
  165. if action in ("spawn", "login", "create_session"):
  166. backoff = float(self.m_cfg.login_backoff)
  167. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  168. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  169. try:
  170. CloudApi.Instance().lock_account(acc.id, int(backoff))
  171. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  172. except Exception as lock_err:
  173. self._log(f"Failed to lock account via cloud API: {lock_err}")
  174. self.m_spawn_cooldown_until = time.time() + backoff
  175. if task:
  176. self._remove_task(task, "spawn account rate limited")
  177. elif action == "query":
  178. # 查询时 429:出口 IP 限流,绝不锁账号,移除当前 task 触发自动换新代理!
  179. self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
  180. if task:
  181. self._remove_task(task, "query ip rate limited")
  182. else:
  183. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  184. if task:
  185. task.next_run = time.time() + 10.0
  186. # 2. 会话过期 / 401
  187. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  188. if action in ("spawn", "login", "create_session"):
  189. self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
  190. self.m_spawn_cooldown_until = time.time() + 5.0
  191. else:
  192. self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
  193. if task:
  194. self._remove_task(task, f"{action} session expired")
  195. # 3. 权限被拒 / WAF 拦截 / 403
  196. elif isinstance(biz_err, PermissionDeniedError):
  197. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  198. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  199. if task:
  200. self._remove_task(task, f"{action} waf blocked")
  201. if action in ("spawn", "login", "create_session"):
  202. self._release_account_safely(acc, reason="WAF Blocked")
  203. self.m_spawn_cooldown_until = time.time() + proxy_cd
  204. # 4. 网络或浏览器超时
  205. elif isinstance(biz_err, NetworkTimeoutError):
  206. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  207. if task:
  208. task.next_run = time.time() + 5.0
  209. if action in ("spawn", "login", "create_session"):
  210. self._release_account_safely(acc, reason="Network Timeout")
  211. self.m_spawn_cooldown_until = time.time() + 10.0
  212. # 5. 兜底策略
  213. else:
  214. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  215. if action in ("spawn", "login", "create_session"):
  216. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  217. self.m_spawn_cooldown_until = time.time() + 10.0
  218. elif task:
  219. task.next_run = time.time() + 10.0
  220. # 6. 统一代理健康指标埋点
  221. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  222. err_type = biz_err.message
  223. if isinstance(biz_err, PermissionDeniedError):
  224. if "403202" in err_type or "cf-mitigated" in err_type.lower() or "waf blocked" in err_type.lower():
  225. err_type = "cf-mitigated (403202)"
  226. ProxyManager.Instance().mark_proxy_result(
  227. proxy=active_plg_cfg.proxy.model_dump(),
  228. plugin_name=self.m_cfg.plugin_config.plugin_name,
  229. action=action,
  230. success=False,
  231. http_code=biz_err.http_status,
  232. error_type=err_type
  233. )
  234. def _is_within_active_hours(self) -> bool:
  235. """
  236. 判断当前是否在允许创建实例的时间段内
  237. """
  238. start_str = self.m_cfg.active_time_start
  239. end_str = self.m_cfg.active_time_end
  240. current_bj_time = datetime.utcnow().time()
  241. start_time = datetime.strptime(start_str, "%H:%M").time()
  242. end_time = datetime.strptime(end_str, "%H:%M").time()
  243. return start_time <= current_bj_time <= end_time
  244. def _monitor_loop(self):
  245. self._log("Monitor loop started.")
  246. self.m_last_group_query_time = 0.0
  247. while not self.m_stop_event.is_set():
  248. try:
  249. time.sleep(1)
  250. now = time.time()
  251. with self.m_lock:
  252. tasks_to_check = list(self.m_tasks)
  253. active_tasks = []
  254. dead_tasks = []
  255. for t in tasks_to_check:
  256. if t.instance.health_check():
  257. active_tasks.append(t)
  258. else:
  259. dead_tasks.append(t)
  260. if dead_tasks:
  261. with self.m_lock:
  262. current_tasks = list(self.m_tasks)
  263. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  264. for t in dead_tasks:
  265. if t in current_tasks:
  266. self._cleanup_task(t, "health check failed")
  267. else:
  268. with self.m_lock:
  269. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  270. if not active_tasks:
  271. continue
  272. avg_interval = self._get_average_interval()
  273. global_gap = max(1.0, avg_interval / len(active_tasks))
  274. active_tasks.sort(key=lambda x: x.next_run)
  275. for task in active_tasks:
  276. if now < task.next_run:
  277. continue
  278. if now - self.m_last_group_query_time < global_gap:
  279. break
  280. apt_types = self.m_cfg.appointment_types
  281. if not apt_types:
  282. continue
  283. weights = [max(0.0, float(item.weight)) for item in apt_types]
  284. if sum(weights) > 0:
  285. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  286. else:
  287. apt_type = random.choice(apt_types)
  288. interval = 30
  289. mode = task.qw_cfg.mode
  290. if mode == QueryWaitMode.Loop:
  291. interval = 1
  292. elif mode == QueryWaitMode.Fixed:
  293. interval = task.qw_cfg.fixed_wait
  294. elif mode == QueryWaitMode.Random:
  295. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  296. self.m_last_group_query_time = now
  297. try:
  298. result = task.instance.query(apt_type)
  299. result.apt_type = apt_type
  300. notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
  301. if result.success:
  302. ttl = self.m_cfg.sentinel.signal_ttl
  303. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  304. payload = {
  305. "group_id": self.m_cfg.identifier,
  306. "apt_type": apt_type.model_dump(),
  307. "query_result": result.to_snapshot_payload(),
  308. "timestamp": time.time()
  309. }
  310. redis_key = self._get_redis_key(apt_type.routing_key)
  311. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  312. task.next_run = time.time() + interval
  313. # 埋点上报 query 成功
  314. if task.instance.config.proxy.port > 0:
  315. ProxyManager.Instance().mark_proxy_result(
  316. proxy=task.instance.config.proxy.model_dump(),
  317. plugin_name=self.m_cfg.plugin_config.plugin_name,
  318. action="query",
  319. success=True,
  320. http_code=200
  321. )
  322. except Exception as e:
  323. self._dispatch_error_policy("query", e, task=task)
  324. break
  325. except Exception as e:
  326. self._dispatch_error_policy("monitor_loop", e)
  327. def _creator_loop(self):
  328. self._log("Creator loop started.")
  329. while not self.m_stop_event.wait(1.0):
  330. if not self._is_within_active_hours():
  331. continue
  332. if time.time() < self.m_spawn_cooldown_until:
  333. continue
  334. with self.m_lock:
  335. current = len(self.m_tasks)
  336. target = self.m_cfg.sentinel.target_instances
  337. if current < target:
  338. self._spawn_sentinel_worker()
  339. def _spawn_sentinel_worker(self):
  340. instance = None
  341. task = None
  342. success = False
  343. plg_cfg = None
  344. try:
  345. plg_cfg = PluginConfig()
  346. plg_cfg.debug = self.m_cfg.debug
  347. plg_cfg.free_config = self.m_cfg.free_config
  348. plg_cfg.session_max_life = self.m_cfg.session_max_life
  349. if not self.m_cfg.need_account:
  350. plg_cfg.account.id = 0
  351. plg_cfg.account.username = "Guest"
  352. else:
  353. acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  354. plg_cfg.account = type(plg_cfg.account)(**acc)
  355. if self.m_cfg.need_proxy:
  356. proxy = ProxyManager.Instance().get_next_proxy(
  357. self.m_cfg.proxy_pool,
  358. self.m_cfg.proxy_cd,
  359. plugin_name=self.m_cfg.plugin_config.plugin_name
  360. )
  361. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  362. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  363. instance.set_log(self.m_logger)
  364. instance.set_config(plg_cfg)
  365. instance.create_session()
  366. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  367. with self.m_lock:
  368. self.m_tasks.append(task)
  369. success = True
  370. if plg_cfg.proxy.port > 0:
  371. ProxyManager.Instance().mark_proxy_result(
  372. proxy=plg_cfg.proxy.model_dump(),
  373. plugin_name=self.m_cfg.plugin_config.plugin_name,
  374. action="spawn",
  375. success=True,
  376. http_code=200
  377. )
  378. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  379. except Exception as e:
  380. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  381. finally:
  382. if not success and instance:
  383. self._cleanup_instance(instance, "spawn failed")