standalone.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394
  1. import os
  2. import time
  3. import random
  4. import threading
  5. from datetime import datetime, timedelta, timezone
  6. from typing import Dict, Callable, Optional
  7. from core_types import (
  8. BookerStandaloneConfig,
  9. QueryWaitMode,
  10. PluginConfig,
  11. Account,
  12. QueryResult,
  13. BookResult,
  14. GroupStatus,
  15. PluginStatus,
  16. utc_now_str,
  17. BizException,
  18. RateLimiteddError,
  19. SessionExpiredOrInvalidError,
  20. PermissionDeniedError,
  21. NetworkTimeoutError,
  22. AccountStateError,
  23. AccountAuthError,
  24. AccountBannedError,
  25. AccountIncompleteError,
  26. AccountNotFoundError,
  27. normalize_error
  28. )
  29. from plugin_factory import PluginFactory
  30. from toolkit.cloud_api import CloudApi
  31. from utils.safe_redis_cli import SafeRedisClient
  32. from toolkit.proxy_manager import ProxyManager
  33. from toolkit.notifier import notify_date_changes, notify_book_result
  34. class StandaloneGCO:
  35. """
  36. Standalone GCO Instance Manager.
  37. Adheres to the unified lifecycle contract of SentinelGCO / BookerGCO (start / stop / update_config).
  38. """
  39. TAG = "STANDALONE"
  40. def __init__(self, cfg: BookerStandaloneConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  41. self.m_cfg = cfg
  42. self.m_logger = logger
  43. self.m_factory = PluginFactory()
  44. self.m_instance = None
  45. self.m_lock = threading.RLock()
  46. self.m_stop_event = threading.Event()
  47. self.m_last_login_time = 0
  48. self.m_next_query_time = 0
  49. self.m_next_keepalive_time = 0
  50. self.m_last_wait_log_time = 0.0
  51. self.m_current_dates = {}
  52. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  53. def _log(self, message: str):
  54. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  55. if self.m_logger:
  56. self.m_logger(f"{prefix} {message}")
  57. else:
  58. print(f"{prefix} {message}")
  59. def _get_wait_interval(self) -> float:
  60. """Calculate wait interval for next query based on configuration."""
  61. mode = self.m_cfg.query_wait.mode
  62. if mode == QueryWaitMode.Loop:
  63. return 1.0
  64. elif mode == QueryWaitMode.Fixed:
  65. return float(self.m_cfg.query_wait.fixed_wait)
  66. elif mode == QueryWaitMode.Random:
  67. return random.uniform(self.m_cfg.query_wait.random_min, self.m_cfg.query_wait.random_max)
  68. return 30.0
  69. def _is_within_active_hours(self) -> bool:
  70. """Determine if current time is within active operating window for instances and queries (UTC)."""
  71. start_str = self.m_cfg.active_time_start
  72. end_str = self.m_cfg.active_time_end
  73. if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
  74. return True
  75. current_utc_time = datetime.now(timezone.utc).time()
  76. start_time = datetime.strptime(start_str, "%H:%M").time()
  77. end_time = datetime.strptime(end_str, "%H:%M").time()
  78. if start_time <= end_time:
  79. return start_time <= current_utc_time <= end_time
  80. else:
  81. return current_utc_time >= start_time or current_utc_time <= end_time
  82. def update_config(self, new_cfg: BookerStandaloneConfig):
  83. """Dynamically update group configuration."""
  84. with self.m_lock:
  85. if self.m_cfg.enable and not new_cfg.enable:
  86. self._log("Config dynamically updated: Group DISABLED.")
  87. elif not self.m_cfg.enable and new_cfg.enable:
  88. self._log("Config dynamically updated: Group ENABLED.")
  89. else:
  90. self._log("Config dynamically updated: Parameters refreshed.")
  91. self.m_cfg = new_cfg
  92. def get_status(self) -> GroupStatus:
  93. with self.m_lock:
  94. instance = self.m_instance
  95. gid = self.m_cfg.identifier
  96. enable = self.m_cfg.enable
  97. plugin_name = self.m_cfg.plugin_config.plugin_name
  98. plugins_status = []
  99. healthy_count = 0
  100. if instance:
  101. st = instance.get_plugin_status()
  102. if st.is_healthy:
  103. healthy_count += 1
  104. plugins_status.append(st)
  105. return GroupStatus(
  106. group_id=gid,
  107. enable=enable,
  108. plugin_name=plugin_name,
  109. total_tasks=1 if instance else 0,
  110. healthy_tasks=healthy_count,
  111. plugins=plugins_status,
  112. updated_at=utc_now_str(),
  113. )
  114. def start(self):
  115. """Non-blocking start: registers plugin and starts background worker thread."""
  116. if not self.m_cfg.enable:
  117. return
  118. self._log("Starting Standalone GCO...")
  119. plugin_name = self.m_cfg.plugin_config.plugin_name
  120. class_name = "".join(part.title() for part in plugin_name.split('_'))
  121. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  122. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  123. worker_name = f"Standalone-{self.m_cfg.identifier}"
  124. threading.Thread(target=self._worker_loop, daemon=True, name=worker_name).start()
  125. def stop(self):
  126. """Gracefully stop background worker and clean up resources."""
  127. self._log("Stopping Standalone GCO...")
  128. self.m_stop_event.set()
  129. self._cleanup_instance("standalone stopped")
  130. def _cleanup_instance(self, reason: str = ""):
  131. with self.m_lock:
  132. if self.m_instance is not None:
  133. try:
  134. self._log(f"🧹 Cleaned up instance. Reason: {reason}")
  135. self.m_instance.cleanup()
  136. except Exception as e:
  137. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  138. finally:
  139. self.m_instance = None
  140. def _init_instance(self):
  141. """Initialize single instance and log in to create a session."""
  142. self._cleanup_instance("recreating new session")
  143. now = time.time()
  144. if now < self.m_last_login_time + self.m_cfg.login_interval:
  145. wait_time = self.m_last_login_time + self.m_cfg.login_interval - now
  146. self._log(f"Waiting {int(wait_time)}s for login interval cooldown...")
  147. if self.m_stop_event.wait(wait_time):
  148. return
  149. plugin_name = self.m_cfg.plugin_config.plugin_name
  150. plg_cfg = PluginConfig()
  151. plg_cfg.debug = self.m_cfg.debug
  152. plg_cfg.account = self.m_cfg.account
  153. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, proxy_cd=600)
  154. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  155. plg_cfg.free_config = self.m_cfg.free_config
  156. plg_cfg.session_max_life = self.m_cfg.session_max_life
  157. instance = self.m_factory.create(self.m_cfg.identifier, plugin_name)
  158. instance.set_log(self.m_logger)
  159. instance.set_config(plg_cfg)
  160. with self.m_lock:
  161. self.m_instance = instance
  162. self._log("Creating session...")
  163. instance.create_session()
  164. self.m_last_login_time = time.time()
  165. interval = self._get_wait_interval()
  166. self.m_next_query_time = time.time() + interval
  167. self.m_next_keepalive_time = time.time() + 10.0
  168. self._log(f"Session created successfully. Initial query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')}).")
  169. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  170. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  171. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  172. return
  173. try:
  174. cloud = CloudApi.Instance()
  175. url = f"{cloud.base_url}/api/account/update"
  176. params = {"account_id": acc.id}
  177. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  178. payload = {
  179. "status": "disable",
  180. "next_use_time": far_future,
  181. "extra_data": {
  182. "disable_reason": reason,
  183. "disabled_at": int(time.time()),
  184. "module": "STANDALONE",
  185. "group": getattr(self.m_cfg, "identifier", "")
  186. }
  187. }
  188. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  189. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  190. except Exception as e:
  191. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  192. try:
  193. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  194. except Exception as lock_err:
  195. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  196. def _dispatch_error_policy(self, action: str, err: Exception) -> None:
  197. """
  198. Standalone Error Recovery & Dispatch Policy Engine:
  199. Executes self-healing actions based on (Action, Exception Domain Type) matrix:
  200. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables account, backs off.
  201. - 429 on create_session/login: Account-level rate limit. Lock account and back off.
  202. - 429 on query: Session alive, egress IP throttled. Do not lock account; rotate proxy.
  203. - session expired / 401: Destroy instance and reschedule re-login in 5s.
  204. - 403 Forbidden / WAF: IP blocked. Rotate proxy and back off.
  205. - Directly maintains logs and timers internally.
  206. """
  207. biz_err = normalize_error(err)
  208. acc = self.m_cfg.account if (self.m_cfg.account and self.m_cfg.account.username not in ("", "Guest")) else None
  209. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  210. if isinstance(biz_err, AccountStateError):
  211. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  212. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  213. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  214. self._cleanup_instance(f"{action} account fatal error: {biz_err.message}")
  215. self.m_last_login_time = time.time()
  216. self.m_next_query_time = time.time() + 3600.0
  217. # 1. Target site rate limit / 429
  218. elif isinstance(biz_err, RateLimiteddError):
  219. if action in ("create_session", "login", "spawn"):
  220. backoff = float(self.m_cfg.login_interval)
  221. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  222. if acc and acc.id > 0:
  223. try:
  224. CloudApi.Instance().lock_account(acc.id, int(backoff))
  225. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  226. except Exception as lock_err:
  227. self._log(f"Failed to lock account via cloud API: {lock_err}")
  228. self.m_last_login_time = time.time()
  229. self._cleanup_instance(f"{action} account rate limited")
  230. elif action == "query":
  231. # 429 during query: Egress IP rate-limited, do not lock account, rotate proxy
  232. self._log("🔄 [Proxy Rate Limited] Query hit 429 (IP throttled, Account preserved). Cleaning instance to rotate proxy...")
  233. self._cleanup_instance("query ip rate limited")
  234. self.m_next_query_time = time.time() + 10.0
  235. else:
  236. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  237. self.m_next_query_time = time.time() + 10.0
  238. # 2. Session expired / 401
  239. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  240. if action in ("create_session", "login", "spawn"):
  241. self._log(f"❌ [Auth Failed] Session creation failed (401 invalid credentials or challenge). Backing off for 10s...")
  242. self.m_last_login_time = time.time() - self.m_cfg.login_interval + 10.0
  243. self._cleanup_instance(f"{action} auth failed")
  244. else:
  245. self._log(f"🔄 [Session Expired] Session died during {action}. Cleaning up instance for re-login in 5s...")
  246. self._cleanup_instance(f"{action} session expired")
  247. self.m_next_query_time = time.time() + 5.0
  248. # 3. Permission denied / WAF blocked / 403
  249. elif isinstance(biz_err, PermissionDeniedError):
  250. self._log(f"🛡️ [WAF Blocked] 403 Forbidden detected during {action}. Cleaning instance to rotate proxy (60s cooldown)...")
  251. self._cleanup_instance(f"{action} waf blocked")
  252. self.m_next_query_time = time.time() + 60.0
  253. # 4. Network or browser timeout
  254. elif isinstance(biz_err, NetworkTimeoutError):
  255. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}. Retrying in 5s...")
  256. self.m_next_query_time = time.time() + 5.0
  257. # 5. Fallback policy
  258. else:
  259. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  260. if action in ("create_session", "worker_loop"):
  261. self.m_last_login_time = time.time() - self.m_cfg.login_interval + 5.0
  262. else:
  263. self.m_next_query_time = time.time() + 10.0
  264. def _worker_loop(self):
  265. """
  266. Background worker daemon loop: keepalive -> schedule check -> query -> booking
  267. """
  268. self._log("Standalone worker loop started.")
  269. while not self.m_stop_event.is_set():
  270. try:
  271. if not self.m_cfg.enable:
  272. self.m_stop_event.wait(1.0)
  273. continue
  274. if not self._is_within_active_hours():
  275. self.m_stop_event.wait(5.0)
  276. continue
  277. # 1. Session initialization and self-healing
  278. if self.m_instance is None:
  279. try:
  280. self._init_instance()
  281. except Exception as e:
  282. self._dispatch_error_policy("create_session", e)
  283. continue
  284. # 2. Health check
  285. if self.m_instance and not self.m_instance.health_check():
  286. self._log("Health check failed. Session dead. Recreating session...")
  287. self._cleanup_instance("session health check failed")
  288. continue
  289. # 3. Query interval wait and periodic keepalive
  290. now = time.time()
  291. if now < self.m_next_query_time:
  292. remaining = self.m_next_query_time - now
  293. next_query_str = datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')
  294. keep_alive_sec = self.m_cfg.keep_alive
  295. # Periodic keepalive
  296. if self.m_instance and now >= self.m_next_keepalive_time:
  297. self.m_instance.keep_alive()
  298. self.m_next_keepalive_time = now + keep_alive_sec
  299. self._log(f"Session keepalive sent. Next query at {next_query_str} (remaining: {remaining:.0f}s)...")
  300. self.m_last_wait_log_time = now
  301. # Periodic wait countdown (logged every 15s)
  302. elif now - self.m_last_wait_log_time >= 15.0 and remaining > 3.0:
  303. self.m_last_wait_log_time = now
  304. self._log(f"Waiting for next query at {next_query_str} (remaining: {remaining:.0f}s)...")
  305. wait_sec = min(remaining, 1.0)
  306. self.m_stop_event.wait(wait_sec)
  307. continue
  308. # 4. Randomly pick a routing_key for query
  309. apt_types = self.m_cfg.appointment_types
  310. if not apt_types:
  311. self.m_stop_event.wait(2.0)
  312. continue
  313. weights = [max(0.0, float(t.weight)) for t in apt_types]
  314. if sum(weights) > 0:
  315. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  316. else:
  317. apt_type = random.choice(apt_types)
  318. self._log(f"Querying slots for {apt_type.routing_key}...")
  319. try:
  320. query_result = self.m_instance.query(apt_type)
  321. query_result.apt_type = apt_type
  322. notify_date_changes(query_result, cache=self.m_current_dates, logger=self._log)
  323. except Exception as e:
  324. self._dispatch_error_policy("query", e)
  325. continue
  326. # 5. Slot found: trigger auto-booking
  327. if query_result.success:
  328. self._log("🔥 SLOT FOUND! Initiating AUTO-BOOKING...")
  329. try:
  330. book_result = self.m_instance.book(query_result, self.m_cfg.user_preferences)
  331. notify_book_result(book_result, logger=self._log)
  332. if book_result.success:
  333. self._log(f"🎉 BOOKING SUCCESSFUL for {apt_type.routing_key}!")
  334. self.m_stop_event.wait(5.0)
  335. except Exception as e:
  336. self._dispatch_error_policy("book", e)
  337. # 6. Recompute and schedule next query and keepalive times
  338. interval = self._get_wait_interval()
  339. now = time.time()
  340. self.m_next_query_time = now + interval
  341. keep_alive_sec = self.m_cfg.keep_alive
  342. self.m_next_keepalive_time = now + keep_alive_sec
  343. self.m_last_wait_log_time = now
  344. self._log(f"Next query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')})...")
  345. except Exception as e:
  346. self._dispatch_error_policy("worker_loop", e)