sentinel.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470
  1. import os
  2. from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
  3. import time
  4. import json
  5. import random
  6. import threading
  7. from datetime import datetime, timedelta, timezone
  8. from typing import List, Dict, Callable, Optional
  9. from core_types import (
  10. GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
  11. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  12. NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError,
  13. AccountIncompleteError, AccountNotFoundError, normalize_error
  14. )
  15. from plugin_factory import PluginFactory
  16. from toolkit.cloud_api import CloudApi
  17. from toolkit.proxy_manager import ProxyManager
  18. from toolkit.notifier import notify_date_changes
  19. from utils.safe_redis_cli import SafeRedisClient
  20. from toolkit.task_executor import TaskExecutor
  21. class SentinelGCO:
  22. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  23. self.m_cfg = cfg
  24. self.m_factory = PluginFactory()
  25. self.m_logger = logger
  26. self.m_tasks: List[Task] = []
  27. self.m_lock = threading.RLock()
  28. self.m_stop_event = threading.Event()
  29. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  30. init_lifecycle(self)
  31. self.m_last_group_query_time = 0.0
  32. self.m_spawn_cooldown_until = 0.0
  33. self.m_current_dates = {}
  34. self._executor = TaskExecutor(cfg.sentinel.max_concurrency, 'Sentinel')
  35. self._background_threads = []
  36. def _log(self, message):
  37. if self.m_logger:
  38. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  39. else:
  40. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  41. def _get_average_interval(self) -> float:
  42. """Calculates the average query interval (in seconds) for the current group."""
  43. mode = self.m_cfg.sentinel.query_wait.mode
  44. if mode == QueryWaitMode.Loop:
  45. return 1.0
  46. elif mode == QueryWaitMode.Fixed:
  47. return max(1.0, float(self.m_cfg.sentinel.query_wait.fixed_wait))
  48. elif mode == QueryWaitMode.Random:
  49. return max(1.0, (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0)
  50. return 30.0
  51. def update_config(self, new_cfg: GroupConfig):
  52. """
  53. Dynamically updates the configuration.
  54. """
  55. with self.m_lock:
  56. if self.m_cfg.enable and not new_cfg.enable:
  57. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  58. elif not self.m_cfg.enable and new_cfg.enable:
  59. self._log("Config dynamically updated: Group ENABLED.")
  60. else:
  61. self._log("Config dynamically updated: Parameters refreshed.")
  62. self.m_cfg = new_cfg
  63. def get_status(self) -> GroupStatus:
  64. with self.m_lock:
  65. tasks = list(self.m_tasks)
  66. gid = self.m_cfg.identifier
  67. enable = self.m_cfg.enable
  68. plugin_name = self.m_cfg.plugin_config.plugin_name
  69. plugins_status = []
  70. healthy_count = 0
  71. for t in tasks:
  72. if t and t.instance:
  73. st = t.instance.get_plugin_status()
  74. if st.is_healthy:
  75. healthy_count += 1
  76. plugins_status.append(st)
  77. return GroupStatus(
  78. group_id=gid,
  79. enable=enable,
  80. plugin_name=plugin_name,
  81. total_tasks=len(tasks),
  82. healthy_tasks=healthy_count,
  83. plugins=plugins_status,
  84. updated_at=utc_now_str(),
  85. )
  86. @start_once
  87. def start(self):
  88. if not self.m_cfg.enable:
  89. return
  90. self._log("Starting Sentinel...")
  91. plugin_name = self.m_cfg.plugin_config.plugin_name
  92. class_name = "".join(part.title() for part in plugin_name.split('_'))
  93. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  94. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  95. for target in (self._monitor_loop, self._creator_loop):
  96. thread = threading.Thread(target=target, daemon=True)
  97. self._background_threads.append(thread)
  98. thread.start()
  99. @serialized_stop
  100. def stop(self):
  101. self._log("Stopping Sentinel...")
  102. self.m_stop_event.set()
  103. deadline = time.monotonic() + self.m_cfg.shutdown_timeout
  104. self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
  105. join_threads(self._background_threads, max(0, deadline - time.monotonic()))
  106. with self.m_lock:
  107. for task in self.m_tasks:
  108. task.removed = True
  109. self._resources.track(task.instance)
  110. self._resources.close(self.m_cfg.cleanup_timeout)
  111. threads_done = self._executor.shutdown(timeout=0.1) and join_threads(self._background_threads, 0.1)
  112. if not self._resources.empty or not threads_done:
  113. raise RuntimeError('Sentinel shutdown incomplete; resources retained for retry')
  114. with self.m_lock:
  115. self.m_tasks.clear()
  116. self.redis_client.close()
  117. def _cleanup_instance(self, instance, reason: str = ""):
  118. if instance is None:
  119. return True
  120. cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
  121. if not cleaned:
  122. self._log(f"Cleanup retained for retry ({reason})")
  123. return cleaned
  124. def _cleanup_task(self, task: Task, reason: str):
  125. if task and task.instance:
  126. with task._operation_lock:
  127. self._cleanup_instance(task.instance, reason)
  128. def _remove_task(self, task: Task, reason: str):
  129. removed = False
  130. with self.m_lock:
  131. if task in self.m_tasks:
  132. task.removed = True
  133. self.m_tasks.remove(task)
  134. removed = True
  135. if removed:
  136. self._cleanup_task(task, reason)
  137. def _get_redis_key(self, routing_key: str) -> str:
  138. return f"vs:signal:{routing_key}"
  139. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  140. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  141. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  142. try:
  143. CloudApi.Instance().lock_account(acc.id, 0)
  144. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  145. except Exception as e:
  146. self._log(f"Failed to release account [{acc.username}]: {e}")
  147. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  148. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  149. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  150. return
  151. try:
  152. cloud = CloudApi.Instance()
  153. url = f"{cloud.base_url}/api/account/update"
  154. params = {"account_id": acc.id}
  155. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  156. payload = {
  157. "status": "disable",
  158. "next_use_time": far_future,
  159. "extra_data": {
  160. "disable_reason": reason,
  161. "disabled_at": int(time.time()),
  162. "module": "SENTINEL",
  163. "group": getattr(self.m_cfg, "identifier", "")
  164. }
  165. }
  166. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  167. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  168. except Exception as e:
  169. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  170. try:
  171. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  172. except Exception as lock_err:
  173. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  174. def _dispatch_error_policy(
  175. self,
  176. action: str,
  177. err: Exception,
  178. task: Optional[Task] = None,
  179. plg_cfg: Optional[PluginConfig] = None
  180. ) -> None:
  181. """
  182. Sentinel error dispatch and self-healing policy engine:
  183. Executes self-healing actions based on the (action, error type) matrix:
  184. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account, never releases back to pool.
  185. - 429 during spawn/login: Account-level rate limit. Locks account and sets m_spawn_cooldown_until backoff.
  186. - 429 during query: Session is valid; egress IP frequency limit. Preserves account, removes task to trigger creator spawning a new instance with a fresh proxy.
  187. - 401 / session expired: Removes broken task.
  188. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  189. - Handles logging and timer backoffs internally.
  190. """
  191. biz_err = normalize_error(err)
  192. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  193. acc = active_plg_cfg.account if active_plg_cfg else None
  194. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  195. if isinstance(biz_err, AccountStateError):
  196. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  197. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  198. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  199. # Never release account back to pool
  200. if task:
  201. self._remove_task(task, f"account fatal error: {biz_err.message}")
  202. if action in ("spawn", "login", "create_session"):
  203. self.m_spawn_cooldown_until = time.time() + 5.0
  204. # 1. Target site rate limit / 429
  205. elif isinstance(biz_err, RateLimiteddError):
  206. if action in ("spawn", "login", "create_session"):
  207. backoff = float(self.m_cfg.login_backoff)
  208. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  209. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  210. try:
  211. CloudApi.Instance().lock_account(acc.id, int(backoff))
  212. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  213. except Exception as lock_err:
  214. self._log(f"Failed to lock account via cloud API: {lock_err}")
  215. self.m_spawn_cooldown_until = time.time() + backoff
  216. if task:
  217. self._remove_task(task, "spawn account rate limited")
  218. elif action == "query":
  219. # 429 during query: egress IP throttled, preserve account, remove current task to trigger auto-rotation to a new proxy
  220. self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
  221. if task:
  222. self._remove_task(task, "query ip rate limited")
  223. else:
  224. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  225. if task:
  226. task.next_run = time.time() + 10.0
  227. # 2. Session expired / 401
  228. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  229. if action in ("spawn", "login", "create_session"):
  230. self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
  231. self.m_spawn_cooldown_until = time.time() + 5.0
  232. else:
  233. self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
  234. if task:
  235. self._remove_task(task, f"{action} session expired")
  236. # 3. Permission denied / WAF blocked / 403
  237. elif isinstance(biz_err, PermissionDeniedError):
  238. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  239. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  240. if task:
  241. self._remove_task(task, f"{action} waf blocked")
  242. if action in ("spawn", "login", "create_session"):
  243. self._release_account_safely(acc, reason="WAF Blocked")
  244. self.m_spawn_cooldown_until = time.time() + proxy_cd
  245. # 4. Network or browser timeout
  246. elif isinstance(biz_err, NetworkTimeoutError):
  247. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  248. if task:
  249. task.next_run = time.time() + 5.0
  250. if action in ("spawn", "login", "create_session"):
  251. self._release_account_safely(acc, reason="Network Timeout")
  252. spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
  253. self.m_spawn_cooldown_until = time.time() + spawn_cd
  254. # 5. Fallback policy
  255. else:
  256. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  257. if action in ("spawn", "login", "create_session"):
  258. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  259. spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
  260. self.m_spawn_cooldown_until = time.time() + spawn_cd
  261. elif task:
  262. task.next_run = time.time() + 10.0
  263. # 6. Proxy health metrics recording
  264. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  265. ProxyManager.Instance().mark_proxy_result(
  266. proxy=active_plg_cfg.proxy.model_dump(),
  267. plugin_name=self.m_cfg.plugin_config.plugin_name,
  268. action=action,
  269. success=False,
  270. http_code=biz_err.http_status,
  271. error_type=biz_err.message
  272. )
  273. def _is_within_active_hours(self) -> bool:
  274. """
  275. Checks if current time is within allowed operating hours (UTC).
  276. """
  277. start_str = self.m_cfg.active_time_start
  278. end_str = self.m_cfg.active_time_end
  279. if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
  280. return True
  281. current_utc_time = datetime.now(timezone.utc).time()
  282. start_time = datetime.strptime(start_str, "%H:%M").time()
  283. end_time = datetime.strptime(end_str, "%H:%M").time()
  284. if start_time <= end_time:
  285. return start_time <= current_utc_time <= end_time
  286. else:
  287. return current_utc_time >= start_time or current_utc_time <= end_time
  288. def _monitor_task(self, task, apt_type):
  289. try:
  290. with task._operation_lock, self._resources.operation(task.instance):
  291. if self.m_stop_event.is_set() or task.removed:
  292. return
  293. now = time.time()
  294. if now >= task.next_health_check:
  295. if not task.instance.health_check():
  296. self._remove_task(task, "health check failed")
  297. return
  298. task.next_health_check = now + 10
  299. if apt_type is None:
  300. return
  301. qw = self.m_cfg.sentinel.query_wait
  302. if qw.mode == QueryWaitMode.Loop:
  303. interval = 1
  304. elif qw.mode == QueryWaitMode.Fixed:
  305. interval = qw.fixed_wait
  306. else:
  307. interval = random.uniform(qw.random_min, qw.random_max)
  308. result = task.instance.query(apt_type)
  309. result.apt_type = apt_type
  310. with self.m_lock:
  311. notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
  312. # Also publish no-slot transitions; never leave a stale positive
  313. # signal until TTL after the site has already withdrawn it.
  314. published_at = time.time()
  315. payload = {"group_id": self.m_cfg.identifier,
  316. "apt_type": apt_type.model_dump(),
  317. "query_result": result.to_snapshot_payload(),
  318. "timestamp": published_at,
  319. "expires_at": published_at + self.m_cfg.sentinel.signal_ttl}
  320. if not self.redis_client.setex(self._get_redis_key(apt_type.routing_key),
  321. self.m_cfg.sentinel.signal_ttl, json.dumps(payload)):
  322. self._log(f'Slot signal publication failed: {apt_type.routing_key}')
  323. task.next_run = time.time() + max(1.0, interval)
  324. if task.instance.config.proxy.port > 0:
  325. ProxyManager.Instance().mark_proxy_result(
  326. proxy=task.instance.config.proxy.model_dump(),
  327. plugin_name=self.m_cfg.plugin_config.plugin_name,
  328. action="query", success=True, http_code=200)
  329. except Exception as exc:
  330. self._dispatch_error_policy("query", exc, task=task)
  331. finally:
  332. with self.m_lock:
  333. task.busy = False
  334. def _schedule_monitor_jobs(self):
  335. with self.m_lock:
  336. tasks = sorted(self.m_tasks, key=lambda t: t.next_run)
  337. gap = max(0.05, self._get_average_interval() / max(1, len(tasks)))
  338. for task in tasks:
  339. if task.busy or task.removed or self.m_stop_event.is_set():
  340. continue
  341. now = time.time()
  342. apt_type = None
  343. apt_types = self.m_cfg.appointment_types
  344. if (apt_types and now >= task.next_run
  345. and now - self.m_last_group_query_time >= gap):
  346. weights = [max(0, apt.weight) for apt in apt_types]
  347. apt_type = (random.choices(apt_types, weights=weights, k=1)[0]
  348. if sum(weights) else random.choice(apt_types))
  349. if apt_type is None and now < task.next_health_check:
  350. continue
  351. task.busy = True
  352. if self._executor.submit(self._monitor_task, task, apt_type):
  353. if apt_type is not None:
  354. self.m_last_group_query_time = now
  355. else:
  356. task.busy = False
  357. def _monitor_loop(self):
  358. self._log("Monitor loop started.")
  359. while not self.m_stop_event.wait(0.05):
  360. try:
  361. if self.m_cfg.enable and self._is_within_active_hours():
  362. self._schedule_monitor_jobs()
  363. except Exception as exc:
  364. self._dispatch_error_policy("monitor_loop", exc)
  365. def _creator_loop(self):
  366. self._log("Creator loop started.")
  367. while not self.m_stop_event.wait(1.0):
  368. if not self._is_within_active_hours():
  369. continue
  370. if time.time() < self.m_spawn_cooldown_until:
  371. continue
  372. with self.m_lock:
  373. current = len(self.m_tasks)
  374. target = self.m_cfg.sentinel.target_instances
  375. if current < target:
  376. self._spawn_sentinel_worker()
  377. def _spawn_sentinel_worker(self):
  378. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  379. return
  380. instance = None
  381. task = None
  382. success = False
  383. plg_cfg = None
  384. try:
  385. plg_cfg = PluginConfig()
  386. plg_cfg.debug = self.m_cfg.debug
  387. plg_cfg.free_config = self.m_cfg.free_config
  388. plg_cfg.session_max_life = self.m_cfg.session_max_life
  389. if not self.m_cfg.need_account:
  390. plg_cfg.account.id = 0
  391. plg_cfg.account.username = "Guest"
  392. else:
  393. acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  394. plg_cfg.account = type(plg_cfg.account)(**acc)
  395. if self.m_cfg.need_proxy:
  396. proxy = ProxyManager.Instance().get_next_proxy(
  397. self.m_cfg.proxy_pool,
  398. self.m_cfg.proxy_cd,
  399. plugin_name=self.m_cfg.plugin_config.plugin_name
  400. )
  401. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  402. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  403. self._resources.track(instance)
  404. with self._resources.operation(instance):
  405. instance.set_log(self.m_logger)
  406. instance.set_config(plg_cfg)
  407. instance.create_session()
  408. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  409. with self.m_lock:
  410. if self.m_stop_event.is_set():
  411. return
  412. self.m_tasks.append(task)
  413. success = True
  414. if plg_cfg.proxy.port > 0:
  415. ProxyManager.Instance().mark_proxy_result(
  416. proxy=plg_cfg.proxy.model_dump(),
  417. plugin_name=self.m_cfg.plugin_config.plugin_name,
  418. action="spawn",
  419. success=True,
  420. http_code=200
  421. )
  422. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  423. except Exception as e:
  424. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  425. finally:
  426. if not success and instance:
  427. self._cleanup_instance(instance, "spawn failed")