sentinel.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime, timedelta, timezone
  7. from typing import List, Dict, Callable, Optional
  8. from core_types import (
  9. GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
  10. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  11. NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError,
  12. AccountIncompleteError, AccountNotFoundError, normalize_error
  13. )
  14. from plugin_factory import PluginFactory
  15. from toolkit.cloud_api import CloudApi
  16. from toolkit.proxy_manager import ProxyManager
  17. from toolkit.notifier import notify_date_changes
  18. from utils.safe_redis_cli import SafeRedisClient
  19. class SentinelGCO:
  20. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  21. self.m_cfg = cfg
  22. self.m_factory = PluginFactory()
  23. self.m_logger = logger
  24. self.m_tasks: List[Task] = []
  25. self.m_lock = threading.RLock()
  26. self.m_stop_event = threading.Event()
  27. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  28. self.m_last_group_query_time = 0.0
  29. self.m_spawn_cooldown_until = 0.0
  30. self.m_current_dates = {}
  31. def _log(self, message):
  32. if self.m_logger:
  33. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  34. else:
  35. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  36. def _get_average_interval(self) -> float:
  37. """Calculates the average query interval (in seconds) for the current group."""
  38. mode = self.m_cfg.sentinel.query_wait.mode
  39. if mode == QueryWaitMode.Loop:
  40. return 1.0
  41. elif mode == QueryWaitMode.Fixed:
  42. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  43. elif mode == QueryWaitMode.Random:
  44. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  45. return 30.0
  46. def update_config(self, new_cfg: GroupConfig):
  47. """
  48. Dynamically updates the configuration.
  49. """
  50. with self.m_lock:
  51. if self.m_cfg.enable and not new_cfg.enable:
  52. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  53. elif not self.m_cfg.enable and new_cfg.enable:
  54. self._log("Config dynamically updated: Group ENABLED.")
  55. else:
  56. self._log("Config dynamically updated: Parameters refreshed.")
  57. self.m_cfg = new_cfg
  58. def get_status(self) -> GroupStatus:
  59. with self.m_lock:
  60. tasks = list(self.m_tasks)
  61. gid = self.m_cfg.identifier
  62. enable = self.m_cfg.enable
  63. plugin_name = self.m_cfg.plugin_config.plugin_name
  64. plugins_status = []
  65. healthy_count = 0
  66. for t in tasks:
  67. if t and t.instance:
  68. st = t.instance.get_plugin_status()
  69. if st.is_healthy:
  70. healthy_count += 1
  71. plugins_status.append(st)
  72. return GroupStatus(
  73. group_id=gid,
  74. enable=enable,
  75. plugin_name=plugin_name,
  76. total_tasks=len(tasks),
  77. healthy_tasks=healthy_count,
  78. plugins=plugins_status,
  79. updated_at=utc_now_str(),
  80. )
  81. def start(self):
  82. if not self.m_cfg.enable:
  83. return
  84. self._log("Starting Sentinel...")
  85. plugin_name = self.m_cfg.plugin_config.plugin_name
  86. class_name = "".join(part.title() for part in plugin_name.split('_'))
  87. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  88. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  89. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  90. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  91. def stop(self):
  92. self._log("Stopping Sentinel...")
  93. self.m_stop_event.set()
  94. with self.m_lock:
  95. tasks_to_cleanup = list(self.m_tasks)
  96. self.m_tasks.clear()
  97. for task in tasks_to_cleanup:
  98. self._cleanup_task(task, "sentinel stopped")
  99. def _cleanup_instance(self, instance, reason: str = ""):
  100. try:
  101. if instance:
  102. self._log(f"🧹 Cleaned up instance. Reason: {reason}")
  103. instance.cleanup()
  104. except Exception as e:
  105. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  106. def _cleanup_task(self, task: Task, reason: str):
  107. if task and task.instance:
  108. self._cleanup_instance(task.instance, reason)
  109. def _remove_task(self, task: Task, reason: str):
  110. removed = False
  111. with self.m_lock:
  112. if task in self.m_tasks:
  113. self.m_tasks.remove(task)
  114. removed = True
  115. if removed:
  116. self._cleanup_task(task, reason)
  117. def _get_redis_key(self, routing_key: str) -> str:
  118. return f"vs:signal:{routing_key}"
  119. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  120. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  121. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  122. try:
  123. CloudApi.Instance().lock_account(acc.id, 0)
  124. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  125. except Exception as e:
  126. self._log(f"Failed to release account [{acc.username}]: {e}")
  127. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  128. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  129. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  130. return
  131. try:
  132. cloud = CloudApi.Instance()
  133. url = f"{cloud.base_url}/api/account/update"
  134. params = {"account_id": acc.id}
  135. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  136. payload = {
  137. "status": "disable",
  138. "next_use_time": far_future,
  139. "extra_data": {
  140. "disable_reason": reason,
  141. "disabled_at": int(time.time()),
  142. "module": "SENTINEL",
  143. "group": getattr(self.m_cfg, "identifier", "")
  144. }
  145. }
  146. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  147. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  148. except Exception as e:
  149. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  150. try:
  151. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  152. except Exception as lock_err:
  153. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  154. def _dispatch_error_policy(
  155. self,
  156. action: str,
  157. err: Exception,
  158. task: Optional[Task] = None,
  159. plg_cfg: Optional[PluginConfig] = None
  160. ) -> None:
  161. """
  162. Sentinel error dispatch and self-healing policy engine:
  163. Executes self-healing actions based on the (action, error type) matrix:
  164. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account, never releases back to pool.
  165. - 429 during spawn/login: Account-level rate limit. Locks account and sets m_spawn_cooldown_until backoff.
  166. - 429 during query: Session is valid; egress IP frequency limit. Preserves account, removes task to trigger creator spawning a new instance with a fresh proxy.
  167. - 401 / session expired: Removes broken task.
  168. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  169. - Handles logging and timer backoffs internally.
  170. """
  171. biz_err = normalize_error(err)
  172. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  173. acc = active_plg_cfg.account if active_plg_cfg else None
  174. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  175. if isinstance(biz_err, AccountStateError):
  176. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  177. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  178. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  179. # Never release account back to pool
  180. if task:
  181. self._remove_task(task, f"account fatal error: {biz_err.message}")
  182. if action in ("spawn", "login", "create_session"):
  183. self.m_spawn_cooldown_until = time.time() + 5.0
  184. # 1. Target site rate limit / 429
  185. elif isinstance(biz_err, RateLimiteddError):
  186. if action in ("spawn", "login", "create_session"):
  187. backoff = float(self.m_cfg.login_backoff)
  188. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  189. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  190. try:
  191. CloudApi.Instance().lock_account(acc.id, int(backoff))
  192. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  193. except Exception as lock_err:
  194. self._log(f"Failed to lock account via cloud API: {lock_err}")
  195. self.m_spawn_cooldown_until = time.time() + backoff
  196. if task:
  197. self._remove_task(task, "spawn account rate limited")
  198. elif action == "query":
  199. # 429 during query: egress IP throttled, preserve account, remove current task to trigger auto-rotation to a new proxy
  200. self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
  201. if task:
  202. self._remove_task(task, "query ip rate limited")
  203. else:
  204. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  205. if task:
  206. task.next_run = time.time() + 10.0
  207. # 2. Session expired / 401
  208. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  209. if action in ("spawn", "login", "create_session"):
  210. self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
  211. self.m_spawn_cooldown_until = time.time() + 5.0
  212. else:
  213. self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
  214. if task:
  215. self._remove_task(task, f"{action} session expired")
  216. # 3. Permission denied / WAF blocked / 403
  217. elif isinstance(biz_err, PermissionDeniedError):
  218. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  219. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  220. if task:
  221. self._remove_task(task, f"{action} waf blocked")
  222. if action in ("spawn", "login", "create_session"):
  223. self._release_account_safely(acc, reason="WAF Blocked")
  224. self.m_spawn_cooldown_until = time.time() + proxy_cd
  225. # 4. Network or browser timeout
  226. elif isinstance(biz_err, NetworkTimeoutError):
  227. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  228. if task:
  229. task.next_run = time.time() + 5.0
  230. if action in ("spawn", "login", "create_session"):
  231. self._release_account_safely(acc, reason="Network Timeout")
  232. self.m_spawn_cooldown_until = time.time() + 10.0
  233. # 5. Fallback policy
  234. else:
  235. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  236. if action in ("spawn", "login", "create_session"):
  237. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  238. self.m_spawn_cooldown_until = time.time() + 10.0
  239. elif task:
  240. task.next_run = time.time() + 10.0
  241. # 6. Proxy health metrics recording
  242. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  243. ProxyManager.Instance().mark_proxy_result(
  244. proxy=active_plg_cfg.proxy.model_dump(),
  245. plugin_name=self.m_cfg.plugin_config.plugin_name,
  246. action=action,
  247. success=False,
  248. http_code=biz_err.http_status,
  249. error_type=biz_err.message
  250. )
  251. def _is_within_active_hours(self) -> bool:
  252. """
  253. Checks if current time is within allowed operating hours (UTC).
  254. """
  255. start_str = self.m_cfg.active_time_start
  256. end_str = self.m_cfg.active_time_end
  257. if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
  258. return True
  259. current_utc_time = datetime.now(timezone.utc).time()
  260. start_time = datetime.strptime(start_str, "%H:%M").time()
  261. end_time = datetime.strptime(end_str, "%H:%M").time()
  262. if start_time <= end_time:
  263. return start_time <= current_utc_time <= end_time
  264. else:
  265. return current_utc_time >= start_time or current_utc_time <= end_time
  266. def _monitor_loop(self):
  267. self._log("Monitor loop started.")
  268. self.m_last_group_query_time = 0.0
  269. while not self.m_stop_event.is_set():
  270. try:
  271. time.sleep(1)
  272. now = time.time()
  273. with self.m_lock:
  274. tasks_to_check = list(self.m_tasks)
  275. active_tasks = []
  276. dead_tasks = []
  277. for t in tasks_to_check:
  278. if t.instance.health_check():
  279. active_tasks.append(t)
  280. else:
  281. dead_tasks.append(t)
  282. if dead_tasks:
  283. with self.m_lock:
  284. for t in dead_tasks:
  285. if t in self.m_tasks:
  286. self.m_tasks.remove(t)
  287. for t in dead_tasks:
  288. self._cleanup_task(t, "health check failed")
  289. if not active_tasks:
  290. continue
  291. avg_interval = self._get_average_interval()
  292. global_gap = max(1.0, avg_interval / len(active_tasks))
  293. active_tasks.sort(key=lambda x: x.next_run)
  294. for task in active_tasks:
  295. if now < task.next_run:
  296. continue
  297. if now - self.m_last_group_query_time < global_gap:
  298. break
  299. apt_types = self.m_cfg.appointment_types
  300. if not apt_types:
  301. continue
  302. weights = [max(0.0, float(item.weight)) for item in apt_types]
  303. if sum(weights) > 0:
  304. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  305. else:
  306. apt_type = random.choice(apt_types)
  307. interval = 30
  308. mode = task.qw_cfg.mode
  309. if mode == QueryWaitMode.Loop:
  310. interval = 1
  311. elif mode == QueryWaitMode.Fixed:
  312. interval = task.qw_cfg.fixed_wait
  313. elif mode == QueryWaitMode.Random:
  314. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  315. self.m_last_group_query_time = now
  316. try:
  317. result = task.instance.query(apt_type)
  318. result.apt_type = apt_type
  319. notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
  320. if result.success:
  321. ttl = self.m_cfg.sentinel.signal_ttl
  322. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  323. payload = {
  324. "group_id": self.m_cfg.identifier,
  325. "apt_type": apt_type.model_dump(),
  326. "query_result": result.to_snapshot_payload(),
  327. "timestamp": time.time()
  328. }
  329. redis_key = self._get_redis_key(apt_type.routing_key)
  330. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  331. task.next_run = time.time() + interval
  332. # Record successful query metrics for proxy
  333. if task.instance.config.proxy.port > 0:
  334. ProxyManager.Instance().mark_proxy_result(
  335. proxy=task.instance.config.proxy.model_dump(),
  336. plugin_name=self.m_cfg.plugin_config.plugin_name,
  337. action="query",
  338. success=True,
  339. http_code=200
  340. )
  341. except Exception as e:
  342. self._dispatch_error_policy("query", e, task=task)
  343. break
  344. except Exception as e:
  345. self._dispatch_error_policy("monitor_loop", e)
  346. def _creator_loop(self):
  347. self._log("Creator loop started.")
  348. while not self.m_stop_event.wait(1.0):
  349. if not self._is_within_active_hours():
  350. continue
  351. if time.time() < self.m_spawn_cooldown_until:
  352. continue
  353. with self.m_lock:
  354. current = len(self.m_tasks)
  355. target = self.m_cfg.sentinel.target_instances
  356. if current < target:
  357. self._spawn_sentinel_worker()
  358. def _spawn_sentinel_worker(self):
  359. instance = None
  360. task = None
  361. success = False
  362. plg_cfg = None
  363. try:
  364. plg_cfg = PluginConfig()
  365. plg_cfg.debug = self.m_cfg.debug
  366. plg_cfg.free_config = self.m_cfg.free_config
  367. plg_cfg.session_max_life = self.m_cfg.session_max_life
  368. if not self.m_cfg.need_account:
  369. plg_cfg.account.id = 0
  370. plg_cfg.account.username = "Guest"
  371. else:
  372. acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  373. plg_cfg.account = type(plg_cfg.account)(**acc)
  374. if self.m_cfg.need_proxy:
  375. proxy = ProxyManager.Instance().get_next_proxy(
  376. self.m_cfg.proxy_pool,
  377. self.m_cfg.proxy_cd,
  378. plugin_name=self.m_cfg.plugin_config.plugin_name
  379. )
  380. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  381. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  382. instance.set_log(self.m_logger)
  383. instance.set_config(plg_cfg)
  384. instance.create_session()
  385. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  386. with self.m_lock:
  387. self.m_tasks.append(task)
  388. success = True
  389. if plg_cfg.proxy.port > 0:
  390. ProxyManager.Instance().mark_proxy_result(
  391. proxy=plg_cfg.proxy.model_dump(),
  392. plugin_name=self.m_cfg.plugin_config.plugin_name,
  393. action="spawn",
  394. success=True,
  395. http_code=200
  396. )
  397. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  398. except Exception as e:
  399. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  400. finally:
  401. if not success and instance:
  402. self._cleanup_instance(instance, "spawn failed")