sentinel.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime, timedelta, timezone
  7. from typing import List, Dict, Callable, Optional
  8. from core_types import (
  9. GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
  10. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  11. NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError,
  12. AccountIncompleteError, AccountNotFoundError, normalize_error
  13. )
  14. from plugin_factory import PluginFactory
  15. from toolkit.cloud_api import CloudApi
  16. from toolkit.proxy_manager import ProxyManager
  17. from toolkit.notifier import notify_date_changes
  18. from utils.safe_redis_cli import SafeRedisClient
  19. class SentinelGCO:
  20. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  21. self.m_cfg = cfg
  22. self.m_factory = PluginFactory()
  23. self.m_logger = logger
  24. self.m_tasks: List[Task] = []
  25. self.m_lock = threading.RLock()
  26. self.m_stop_event = threading.Event()
  27. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  28. self.m_last_group_query_time = 0.0
  29. self.m_spawn_cooldown_until = 0.0
  30. self.m_current_dates = {}
  31. def _log(self, message):
  32. if self.m_logger:
  33. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  34. else:
  35. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  36. def _get_average_interval(self) -> float:
  37. """Calculates the average query interval (in seconds) for the current group."""
  38. mode = self.m_cfg.sentinel.query_wait.mode
  39. if mode == QueryWaitMode.Loop:
  40. return 1.0
  41. elif mode == QueryWaitMode.Fixed:
  42. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  43. elif mode == QueryWaitMode.Random:
  44. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  45. return 30.0
  46. def update_config(self, new_cfg: GroupConfig):
  47. """
  48. Dynamically updates the configuration.
  49. """
  50. with self.m_lock:
  51. if self.m_cfg.enable and not new_cfg.enable:
  52. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  53. elif not self.m_cfg.enable and new_cfg.enable:
  54. self._log("Config dynamically updated: Group ENABLED.")
  55. else:
  56. self._log("Config dynamically updated: Parameters refreshed.")
  57. self.m_cfg = new_cfg
  58. def get_status(self) -> GroupStatus:
  59. with self.m_lock:
  60. tasks = list(self.m_tasks)
  61. gid = self.m_cfg.identifier
  62. enable = self.m_cfg.enable
  63. plugin_name = self.m_cfg.plugin_config.plugin_name
  64. plugins_status = []
  65. healthy_count = 0
  66. for t in tasks:
  67. if t and t.instance:
  68. st = t.instance.get_plugin_status()
  69. if st.is_healthy:
  70. healthy_count += 1
  71. plugins_status.append(st)
  72. return GroupStatus(
  73. group_id=gid,
  74. enable=enable,
  75. plugin_name=plugin_name,
  76. total_tasks=len(tasks),
  77. healthy_tasks=healthy_count,
  78. plugins=plugins_status,
  79. updated_at=utc_now_str(),
  80. )
  81. def start(self):
  82. if not self.m_cfg.enable:
  83. return
  84. self._log("Starting Sentinel...")
  85. plugin_name = self.m_cfg.plugin_config.plugin_name
  86. class_name = "".join(part.title() for part in plugin_name.split('_'))
  87. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  88. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  89. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  90. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  91. def stop(self):
  92. self._log("Stopping Sentinel...")
  93. self.m_stop_event.set()
  94. with self.m_lock:
  95. tasks_to_cleanup = list(self.m_tasks)
  96. self.m_tasks.clear()
  97. for task in tasks_to_cleanup:
  98. self._cleanup_task(task, "sentinel stopped")
  99. def _cleanup_instance(self, instance, reason: str = ""):
  100. try:
  101. if instance:
  102. self._log(f"🧹 Cleaned up instance. Reason: {reason}")
  103. instance.cleanup()
  104. except Exception as e:
  105. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  106. def _cleanup_task(self, task: Task, reason: str):
  107. if task and task.instance:
  108. self._cleanup_instance(task.instance, reason)
  109. def _remove_task(self, task: Task, reason: str):
  110. removed = False
  111. with self.m_lock:
  112. if task in self.m_tasks:
  113. self.m_tasks.remove(task)
  114. removed = True
  115. if removed:
  116. self._cleanup_task(task, reason)
  117. def _get_redis_key(self, routing_key: str) -> str:
  118. return f"vs:signal:{routing_key}"
  119. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  120. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  121. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  122. try:
  123. CloudApi.Instance().lock_account(acc.id, 0)
  124. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  125. except Exception as e:
  126. self._log(f"Failed to release account [{acc.username}]: {e}")
  127. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  128. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  129. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  130. return
  131. try:
  132. cloud = CloudApi.Instance()
  133. url = f"{cloud.base_url}/api/account/update"
  134. params = {"account_id": acc.id}
  135. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  136. payload = {
  137. "status": "disable",
  138. "next_use_time": far_future,
  139. "extra_data": {
  140. "disable_reason": reason,
  141. "disabled_at": int(time.time()),
  142. "module": "SENTINEL",
  143. "group": getattr(self.m_cfg, "identifier", "")
  144. }
  145. }
  146. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  147. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  148. except Exception as e:
  149. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  150. try:
  151. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  152. except Exception as lock_err:
  153. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  154. def _dispatch_error_policy(
  155. self,
  156. action: str,
  157. err: Exception,
  158. task: Optional[Task] = None,
  159. plg_cfg: Optional[PluginConfig] = None
  160. ) -> None:
  161. """
  162. Sentinel error dispatch and self-healing policy engine:
  163. Executes self-healing actions based on the (action, error type) matrix:
  164. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account, never releases back to pool.
  165. - 429 during spawn/login: Account-level rate limit. Locks account and sets m_spawn_cooldown_until backoff.
  166. - 429 during query: Session is valid; egress IP frequency limit. Preserves account, removes task to trigger creator spawning a new instance with a fresh proxy.
  167. - 401 / session expired: Removes broken task.
  168. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  169. - Handles logging and timer backoffs internally.
  170. """
  171. biz_err = normalize_error(err)
  172. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  173. acc = active_plg_cfg.account if active_plg_cfg else None
  174. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  175. if isinstance(biz_err, AccountStateError):
  176. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  177. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  178. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  179. # Never release account back to pool
  180. if task:
  181. self._remove_task(task, f"account fatal error: {biz_err.message}")
  182. if action in ("spawn", "login", "create_session"):
  183. self.m_spawn_cooldown_until = time.time() + 5.0
  184. # 1. Target site rate limit / 429
  185. elif isinstance(biz_err, RateLimiteddError):
  186. if action in ("spawn", "login", "create_session"):
  187. backoff = float(self.m_cfg.login_backoff)
  188. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  189. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  190. try:
  191. CloudApi.Instance().lock_account(acc.id, int(backoff))
  192. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  193. except Exception as lock_err:
  194. self._log(f"Failed to lock account via cloud API: {lock_err}")
  195. self.m_spawn_cooldown_until = time.time() + backoff
  196. if task:
  197. self._remove_task(task, "spawn account rate limited")
  198. elif action == "query":
  199. # 429 during query: egress IP throttled, preserve account, remove current task to trigger auto-rotation to a new proxy
  200. self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
  201. if task:
  202. self._remove_task(task, "query ip rate limited")
  203. else:
  204. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  205. if task:
  206. task.next_run = time.time() + 10.0
  207. # 2. Session expired / 401
  208. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  209. if action in ("spawn", "login", "create_session"):
  210. self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
  211. self.m_spawn_cooldown_until = time.time() + 5.0
  212. else:
  213. self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
  214. if task:
  215. self._remove_task(task, f"{action} session expired")
  216. # 3. Permission denied / WAF blocked / 403
  217. elif isinstance(biz_err, PermissionDeniedError):
  218. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  219. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  220. if task:
  221. self._remove_task(task, f"{action} waf blocked")
  222. if action in ("spawn", "login", "create_session"):
  223. self._release_account_safely(acc, reason="WAF Blocked")
  224. self.m_spawn_cooldown_until = time.time() + proxy_cd
  225. # 4. Network or browser timeout
  226. elif isinstance(biz_err, NetworkTimeoutError):
  227. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  228. if task:
  229. task.next_run = time.time() + 5.0
  230. if action in ("spawn", "login", "create_session"):
  231. self._release_account_safely(acc, reason="Network Timeout")
  232. self.m_spawn_cooldown_until = time.time() + 10.0
  233. # 5. Fallback policy
  234. else:
  235. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  236. if action in ("spawn", "login", "create_session"):
  237. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  238. self.m_spawn_cooldown_until = time.time() + 10.0
  239. elif task:
  240. task.next_run = time.time() + 10.0
  241. # 6. Proxy health metrics recording
  242. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  243. ProxyManager.Instance().mark_proxy_result(
  244. proxy=active_plg_cfg.proxy.model_dump(),
  245. plugin_name=self.m_cfg.plugin_config.plugin_name,
  246. action=action,
  247. success=False,
  248. http_code=biz_err.http_status,
  249. error_type=biz_err.message
  250. )
  251. def _is_within_active_hours(self) -> bool:
  252. """
  253. Checks if current time is within allowed operating hours.
  254. """
  255. start_str = self.m_cfg.active_time_start
  256. end_str = self.m_cfg.active_time_end
  257. current_bj_time = datetime.utcnow().time()
  258. start_time = datetime.strptime(start_str, "%H:%M").time()
  259. end_time = datetime.strptime(end_str, "%H:%M").time()
  260. return start_time <= current_bj_time <= end_time
  261. def _monitor_loop(self):
  262. self._log("Monitor loop started.")
  263. self.m_last_group_query_time = 0.0
  264. while not self.m_stop_event.is_set():
  265. try:
  266. time.sleep(1)
  267. now = time.time()
  268. with self.m_lock:
  269. tasks_to_check = list(self.m_tasks)
  270. active_tasks = []
  271. dead_tasks = []
  272. for t in tasks_to_check:
  273. if t.instance.health_check():
  274. active_tasks.append(t)
  275. else:
  276. dead_tasks.append(t)
  277. if dead_tasks:
  278. with self.m_lock:
  279. for t in dead_tasks:
  280. if t in self.m_tasks:
  281. self.m_tasks.remove(t)
  282. for t in dead_tasks:
  283. self._cleanup_task(t, "health check failed")
  284. if not active_tasks:
  285. continue
  286. avg_interval = self._get_average_interval()
  287. global_gap = max(1.0, avg_interval / len(active_tasks))
  288. active_tasks.sort(key=lambda x: x.next_run)
  289. for task in active_tasks:
  290. if now < task.next_run:
  291. continue
  292. if now - self.m_last_group_query_time < global_gap:
  293. break
  294. apt_types = self.m_cfg.appointment_types
  295. if not apt_types:
  296. continue
  297. weights = [max(0.0, float(item.weight)) for item in apt_types]
  298. if sum(weights) > 0:
  299. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  300. else:
  301. apt_type = random.choice(apt_types)
  302. interval = 30
  303. mode = task.qw_cfg.mode
  304. if mode == QueryWaitMode.Loop:
  305. interval = 1
  306. elif mode == QueryWaitMode.Fixed:
  307. interval = task.qw_cfg.fixed_wait
  308. elif mode == QueryWaitMode.Random:
  309. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  310. self.m_last_group_query_time = now
  311. try:
  312. result = task.instance.query(apt_type)
  313. result.apt_type = apt_type
  314. notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
  315. if result.success:
  316. ttl = self.m_cfg.sentinel.signal_ttl
  317. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  318. payload = {
  319. "group_id": self.m_cfg.identifier,
  320. "apt_type": apt_type.model_dump(),
  321. "query_result": result.to_snapshot_payload(),
  322. "timestamp": time.time()
  323. }
  324. redis_key = self._get_redis_key(apt_type.routing_key)
  325. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  326. task.next_run = time.time() + interval
  327. # Record successful query metrics for proxy
  328. if task.instance.config.proxy.port > 0:
  329. ProxyManager.Instance().mark_proxy_result(
  330. proxy=task.instance.config.proxy.model_dump(),
  331. plugin_name=self.m_cfg.plugin_config.plugin_name,
  332. action="query",
  333. success=True,
  334. http_code=200
  335. )
  336. except Exception as e:
  337. self._dispatch_error_policy("query", e, task=task)
  338. break
  339. except Exception as e:
  340. self._dispatch_error_policy("monitor_loop", e)
  341. def _creator_loop(self):
  342. self._log("Creator loop started.")
  343. while not self.m_stop_event.wait(1.0):
  344. if not self._is_within_active_hours():
  345. continue
  346. if time.time() < self.m_spawn_cooldown_until:
  347. continue
  348. with self.m_lock:
  349. current = len(self.m_tasks)
  350. target = self.m_cfg.sentinel.target_instances
  351. if current < target:
  352. self._spawn_sentinel_worker()
  353. def _spawn_sentinel_worker(self):
  354. instance = None
  355. task = None
  356. success = False
  357. plg_cfg = None
  358. try:
  359. plg_cfg = PluginConfig()
  360. plg_cfg.debug = self.m_cfg.debug
  361. plg_cfg.free_config = self.m_cfg.free_config
  362. plg_cfg.session_max_life = self.m_cfg.session_max_life
  363. if not self.m_cfg.need_account:
  364. plg_cfg.account.id = 0
  365. plg_cfg.account.username = "Guest"
  366. else:
  367. acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  368. plg_cfg.account = type(plg_cfg.account)(**acc)
  369. if self.m_cfg.need_proxy:
  370. proxy = ProxyManager.Instance().get_next_proxy(
  371. self.m_cfg.proxy_pool,
  372. self.m_cfg.proxy_cd,
  373. plugin_name=self.m_cfg.plugin_config.plugin_name
  374. )
  375. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  376. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  377. instance.set_log(self.m_logger)
  378. instance.set_config(plg_cfg)
  379. instance.create_session()
  380. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  381. with self.m_lock:
  382. self.m_tasks.append(task)
  383. success = True
  384. if plg_cfg.proxy.port > 0:
  385. ProxyManager.Instance().mark_proxy_result(
  386. proxy=plg_cfg.proxy.model_dump(),
  387. plugin_name=self.m_cfg.plugin_config.plugin_name,
  388. action="spawn",
  389. success=True,
  390. http_code=200
  391. )
  392. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  393. except Exception as e:
  394. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  395. finally:
  396. if not success and instance:
  397. self._cleanup_instance(instance, "spawn failed")