booker.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899
  1. import os
  2. from copy import deepcopy
  3. from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
  4. import time
  5. import json
  6. import threading
  7. import random
  8. from datetime import datetime, timedelta, timezone
  9. from typing import List, Dict, Callable, Optional
  10. from core_types import (
  11. GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult,
  12. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
  13. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  14. NetworkTimeoutError, BizLogicError, AccountStateError, AccountAuthError,
  15. AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error
  16. )
  17. from plugin_factory import PluginFactory
  18. from toolkit.cloud_api import CloudApi
  19. from toolkit.proxy_manager import ProxyManager
  20. from toolkit.task_executor import TaskExecutor
  21. from utils.booking_outbox import BookingOutbox
  22. from toolkit.notifier import notify_book_result
  23. from utils.safe_redis_cli import SafeRedisClient
  24. class BaseBookerGCO:
  25. """
  26. Base Booker class encapsulating shared infrastructure, lifecycle management, and logic loops.
  27. """
  28. TAG = "BOOKER"
  29. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  30. self.m_cfg = cfg
  31. self.m_factory = PluginFactory()
  32. self.m_logger = logger
  33. self.m_tasks: List[Task] = []
  34. self.m_lock = threading.RLock()
  35. self.m_stop_event = threading.Event()
  36. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  37. init_lifecycle(self)
  38. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  39. self.m_last_group_query_time = 0.0
  40. self.m_spawn_cooldown_until = 0.0
  41. self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking")
  42. self._leased_orders = set()
  43. self._pending_results = {}
  44. self._outbox = BookingOutbox()
  45. self._background_threads = []
  46. self._heartbeat_stop_event = threading.Event()
  47. def _log(self, message):
  48. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  49. if self.m_logger:
  50. self.m_logger(f'{prefix} {message}')
  51. else:
  52. print(f'{prefix} {message}')
  53. def get_status(self) -> GroupStatus:
  54. with self.m_lock:
  55. tasks = list(self.m_tasks)
  56. gid = self.m_cfg.identifier
  57. enable = self.m_cfg.enable
  58. plugin_name = self.m_cfg.plugin_config.plugin_name
  59. plugins_status = []
  60. healthy_count = 0
  61. for t in tasks:
  62. if t and t.instance:
  63. st = t.instance.get_plugin_status()
  64. if st.is_healthy:
  65. healthy_count += 1
  66. plugins_status.append(st)
  67. return GroupStatus(
  68. group_id=gid,
  69. enable=enable,
  70. plugin_name=plugin_name,
  71. total_tasks=len(tasks),
  72. healthy_tasks=healthy_count,
  73. plugins=plugins_status,
  74. updated_at=utc_now_str(),
  75. )
  76. @start_once
  77. def start(self):
  78. if not self.m_cfg.enable:
  79. return
  80. self._log(f"Starting {self.TAG}...")
  81. plugin_name = self.m_cfg.plugin_config.plugin_name
  82. class_name = "".join(part.title() for part in plugin_name.split('_'))
  83. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  84. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  85. if plugin_name:
  86. self.m_factory.ensure_booking_supported(plugin_name)
  87. for target in (self._booking_trigger_loop, self._creator_loop,
  88. self._maintain_loop, self._heartbeat_loop, self._success_sync_loop):
  89. thread = threading.Thread(target=target, daemon=True, name=target.__name__)
  90. self._background_threads.append(thread)
  91. thread.start()
  92. self._start_additional_threads()
  93. def _start_additional_threads(self):
  94. """Hook for subclasses to start additional background threads."""
  95. pass
  96. @serialized_stop
  97. def stop(self):
  98. self._log("Stopping Booker...")
  99. self.m_stop_event.set()
  100. deadline = time.monotonic() + self.m_cfg.shutdown_timeout
  101. self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
  102. workers = [t for t in self._background_threads if t.name != '_heartbeat_loop']
  103. join_threads(workers, max(0, deadline - time.monotonic()))
  104. with self.m_lock:
  105. for task in self.m_tasks:
  106. task.removed = True
  107. self._resources.track(task.instance)
  108. self._resources.close(self.m_cfg.cleanup_timeout)
  109. threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1)
  110. if threads_done and not self._pending_results:
  111. self._heartbeat_stop_event.set()
  112. threads_done = join_threads(
  113. [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2)
  114. if not self._resources.empty or not threads_done or self._pending_results:
  115. raise RuntimeError('Booker shutdown incomplete; resources retained for retry')
  116. with self.m_lock:
  117. self.m_tasks.clear()
  118. self._on_all_tasks_cleaned()
  119. self._leased_orders.clear()
  120. self.redis_client.close()
  121. def update_config(self, new_cfg: GroupConfig):
  122. """Dynamically updates the configuration."""
  123. with self.m_lock:
  124. if self.m_cfg.enable and not new_cfg.enable:
  125. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  126. elif not self.m_cfg.enable and new_cfg.enable:
  127. self._log("Config dynamically updated: Group ENABLED.")
  128. else:
  129. self._log("Config dynamically updated: Parameters refreshed.")
  130. self.m_cfg = new_cfg
  131. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  132. if instance is None:
  133. return True
  134. cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
  135. if not cleaned:
  136. self._log(f"Cleanup retained for retry ({reason})")
  137. return cleaned
  138. def _cleanup_task(self, task: Task, reason: str = ""):
  139. if task and task.instance:
  140. with task._operation_lock:
  141. self._cleanup_instance(task.instance, reason, task.task_ref)
  142. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  143. removed = False
  144. with self.m_lock:
  145. if task in self.m_tasks:
  146. task.removed = True
  147. self.m_tasks.remove(task)
  148. removed = True
  149. if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
  150. self._leased_orders.discard(str(task.task_ref))
  151. self._on_task_removed(task)
  152. if cleanup and removed:
  153. self._cleanup_task(task, reason)
  154. return removed
  155. def _on_task_removed(self, task: Task):
  156. """Hook for subclasses when a task is removed (e.g. cleaning up caches)."""
  157. pass
  158. def _cleanup_all_tasks(self, reason: str = ""):
  159. with self.m_lock:
  160. tasks = list(self.m_tasks)
  161. self.m_tasks.clear()
  162. for task in tasks:
  163. task.removed = True
  164. if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
  165. self._leased_orders.discard(str(task.task_ref))
  166. self._on_all_tasks_cleaned()
  167. for task in tasks:
  168. self._cleanup_task(task, reason)
  169. def _on_all_tasks_cleaned(self):
  170. """Hook for subclasses when all tasks are cleared."""
  171. pass
  172. def _get_redis_key(self, routing_key: str) -> str:
  173. return f"vs:signal:{routing_key}"
  174. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  175. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  176. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  177. try:
  178. CloudApi.Instance().lock_account(acc.id, 0)
  179. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  180. except Exception as e:
  181. self._log(f"Failed to release account [{acc.username}]: {e}")
  182. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  183. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  184. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  185. return
  186. try:
  187. cloud = CloudApi.Instance()
  188. url = f"{cloud.base_url}/api/account/update"
  189. params = {"account_id": acc.id}
  190. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  191. payload = {
  192. "status": "disable",
  193. "next_use_time": far_future,
  194. "extra_data": {
  195. "disable_reason": reason,
  196. "disabled_at": int(time.time()),
  197. "module": self.TAG,
  198. "group": getattr(self.m_cfg, "identifier", "")
  199. }
  200. }
  201. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  202. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  203. except Exception as e:
  204. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  205. try:
  206. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  207. except Exception as lock_err:
  208. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  209. def _dispatch_error_policy(
  210. self,
  211. action: str,
  212. err: Exception,
  213. task: Optional[Task] = None,
  214. plg_cfg: Optional[PluginConfig] = None,
  215. task_id: Optional[int] = None
  216. ) -> None:
  217. """
  218. Booker error dispatch and self-healing policy engine:
  219. Executes self-healing actions based on the (action, error type) matrix:
  220. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account or fails order task, never releases back to pool.
  221. - 429 during spawn/login: Account-level rate limit. Locks account, updates tracker & cooldown.
  222. - 429 during book: Booking slot rate limit or egress IP throttle. Preserves account, removes task to rotate proxy.
  223. - 401 / session expired: Session invalidated. Removes task.
  224. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  225. - Handles logging and timer backoffs internally.
  226. """
  227. biz_err = normalize_error(err)
  228. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  229. acc = active_plg_cfg.account if active_plg_cfg else None
  230. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  231. if isinstance(biz_err, AccountStateError):
  232. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  233. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  234. # If bound order task, update VAS task status to failed
  235. if effective_task_id is not None:
  236. try:
  237. CloudApi.Instance().update_vas_task(str(effective_task_id), {
  238. "status": "failed",
  239. "fail_reason": f"Account fatal error: {biz_err.message} (code={biz_err.code})"
  240. })
  241. self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
  242. except Exception as update_err:
  243. self._log(f"Failed to update VAS task status: {update_err}")
  244. else:
  245. self.redis_client.zrem(self.m_tracker_key, effective_task_id)
  246. # If pool account, permanently disable it in cloud
  247. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  248. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  249. # Never release account back to pool
  250. if task:
  251. self._remove_task(task, f"account fatal error: {biz_err.message}")
  252. if action in ("spawn", "login", "create_session"):
  253. self.m_spawn_cooldown_until = time.time() + 5.0
  254. # 1. Target site rate limit / 429
  255. elif isinstance(biz_err, RateLimiteddError):
  256. if action in ("spawn", "login", "create_session"):
  257. backoff = float(self.m_cfg.login_backoff)
  258. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  259. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  260. try:
  261. CloudApi.Instance().lock_account(acc.id, int(backoff))
  262. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  263. except Exception as lock_err:
  264. self._log(f"Failed to lock account via cloud API: {lock_err}")
  265. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  266. if effective_task_id is not None:
  267. self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
  268. self.m_spawn_cooldown_until = time.time() + backoff
  269. if task:
  270. self._remove_task(task, "spawn account rate limited")
  271. elif action == "book":
  272. # 429 during booking: slot rate limit or IP restriction, preserve account and remove task to rotate proxy
  273. self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
  274. if task:
  275. self._remove_task(task, "booking rate limited")
  276. else:
  277. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  278. if task:
  279. self._remove_task(task, f"{action} rate limited")
  280. # 2. Session expired / 401
  281. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  282. if action in ("spawn", "login", "create_session"):
  283. self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
  284. self.m_spawn_cooldown_until = time.time() + 5.0
  285. else:
  286. self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
  287. if task:
  288. self._remove_task(task, f"{action} session expired")
  289. # 3. Permission denied / WAF blocked / 403
  290. elif isinstance(biz_err, PermissionDeniedError):
  291. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  292. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  293. if task:
  294. self._remove_task(task, f"{action} waf blocked")
  295. if action in ("spawn", "login", "create_session"):
  296. self._release_account_safely(acc, reason="WAF Blocked")
  297. self.m_spawn_cooldown_until = time.time() + proxy_cd
  298. # 4. Network or browser timeout
  299. elif isinstance(biz_err, NetworkTimeoutError):
  300. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  301. if task:
  302. task.next_run = time.time() + 5.0
  303. if action in ("spawn", "login", "create_session"):
  304. self._release_account_safely(acc, reason="Network Timeout")
  305. spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
  306. self.m_spawn_cooldown_until = time.time() + spawn_cd
  307. # 5. Fallback policy
  308. else:
  309. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  310. if action in ("spawn", "login", "create_session"):
  311. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  312. spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
  313. self.m_spawn_cooldown_until = time.time() + spawn_cd
  314. elif task:
  315. task.next_run = time.time() + 10.0
  316. # 6. Proxy health metrics recording
  317. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  318. ProxyManager.Instance().mark_proxy_result(
  319. proxy=active_plg_cfg.proxy.model_dump(),
  320. plugin_name=self.m_cfg.plugin_config.plugin_name,
  321. action=action,
  322. success=False,
  323. http_code=biz_err.http_status,
  324. error_type=biz_err.message
  325. )
  326. def _is_within_active_hours(self) -> bool:
  327. """Checks if current time is within allowed operating hours (UTC)."""
  328. start_str = self.m_cfg.active_time_start
  329. end_str = self.m_cfg.active_time_end
  330. if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
  331. return True
  332. current_utc_time = datetime.now(timezone.utc).time()
  333. start_time = datetime.strptime(start_str, "%H:%M").time()
  334. end_time = datetime.strptime(end_str, "%H:%M").time()
  335. if start_time <= end_time:
  336. return start_time <= current_utc_time <= end_time
  337. else:
  338. return current_utc_time >= start_time or current_utc_time <= end_time
  339. def _lease_order(self, task_id):
  340. with self.m_lock:
  341. self._leased_orders.add(str(task_id))
  342. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120})
  343. def _renew_leases(self):
  344. with self.m_lock:
  345. if self._leased_orders:
  346. self.redis_client.bulk_zadd(self.m_tracker_key, {
  347. tid: time.time() + 120 for tid in self._leased_orders})
  348. def _heartbeat_loop(self):
  349. # No browser calls here: long OTP/HTTP waits must not stop order leases.
  350. while not self._heartbeat_stop_event.wait(10):
  351. self._renew_leases()
  352. def _sync_successes(self):
  353. try:
  354. with self.m_lock:
  355. pending = dict(self._pending_results)
  356. for task_id, payload in pending.items():
  357. saved = False
  358. try:
  359. self._outbox.put(task_id, self.m_tracker_key, payload)
  360. saved = True
  361. except Exception as exc:
  362. self._log(f"Local success journal unavailable for task={task_id}: {exc}")
  363. saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved
  364. if saved:
  365. with self.m_lock:
  366. self._pending_results.pop(task_id, None)
  367. self._leased_orders.discard(task_id)
  368. self._outbox.flush(CloudApi.Instance(), self.redis_client._client,
  369. self._log, tracker=self.m_tracker_key)
  370. except Exception as exc:
  371. self._log(f"Success outbox retry failed: {exc}")
  372. def _success_sync_loop(self):
  373. while not self.m_stop_event.is_set() or self._pending_results:
  374. self._sync_successes()
  375. if self.m_stop_event.is_set() and not self._pending_results:
  376. break
  377. if self.m_stop_event.is_set():
  378. self._heartbeat_stop_event.wait(5)
  379. else:
  380. self.m_stop_event.wait(5)
  381. def _maintain_task(self, task):
  382. try:
  383. with task._operation_lock, self._resources.operation(task.instance):
  384. if task.removed or self.m_stop_event.is_set():
  385. return
  386. task.instance.keep_alive()
  387. if task.instance.health_check():
  388. task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5))
  389. else:
  390. self._remove_task(task, "unhealthy or keep-alive failed")
  391. except Exception as exc:
  392. task.next_remote_ping = time.time() + 5
  393. self._dispatch_error_policy("keep_alive", exc, task=task)
  394. finally:
  395. with self.m_lock:
  396. task.busy = False
  397. def _schedule_maintenance(self):
  398. with self.m_lock:
  399. if self.m_stop_event.is_set():
  400. return
  401. for task in self.m_tasks:
  402. if task.busy or task.removed or time.time() < task.next_remote_ping:
  403. continue
  404. task.busy = True
  405. if not self._executor.submit(self._maintain_task, task):
  406. task.busy = False
  407. def _maintain_loop(self):
  408. while not self.m_stop_event.wait(1):
  409. self._schedule_maintenance()
  410. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  411. """Checks if available dates in query_result match task preferences (defaults to True)."""
  412. return True
  413. def _run_book_job(self, task, query_result):
  414. try:
  415. with task._operation_lock, self._resources.operation(task.instance):
  416. if not task.removed and not self.m_stop_event.is_set():
  417. self._execute_book_job(task, query_result)
  418. finally:
  419. with self.m_lock:
  420. task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown)
  421. task.busy = False
  422. def _poll_booking_signals(self):
  423. now = time.time()
  424. with self.m_lock:
  425. ready = [t for t in self.m_tasks if not t.busy and not t.removed
  426. and t.book_allowed and now >= t.next_run]
  427. keys = {key for t in ready for key in t.acceptable_routing_keys}
  428. apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys]
  429. if not apt_types:
  430. return
  431. redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types]
  432. signals = self.redis_client.mget(redis_keys)
  433. for apt_type, raw_data in zip(apt_types, signals):
  434. if not raw_data:
  435. continue
  436. try:
  437. data = json.loads(raw_data)
  438. result = QueryResult.model_validate(data['query_result'])
  439. result.apt_type = AppointmentType.model_validate(data['apt_type'])
  440. if result.apt_type.routing_key != apt_type.routing_key:
  441. raise ValueError('signal routing key mismatch')
  442. if result.availability_status not in (AvailabilityStatus.Available, AvailabilityStatus.Waitlist):
  443. continue
  444. if data.get('expires_at', float('inf')) <= time.time():
  445. continue
  446. except Exception as exc:
  447. # A producer may have replaced this value since MGET. Do not
  448. # delete a newer signal based on an old malformed payload.
  449. self._log(f"Invalid signal for {apt_type.routing_key}: {exc}")
  450. continue
  451. with self.m_lock:
  452. for task in ready:
  453. if (task.busy or task.removed or self.m_stop_event.is_set()
  454. or apt_type.routing_key not in task.acceptable_routing_keys
  455. or not self._is_date_of_interest(task, result)):
  456. continue
  457. task.busy = True
  458. if not self._executor.submit(self._run_book_job, task, result):
  459. task.busy = False
  460. def _booking_trigger_loop(self):
  461. self._log("Trigger loop started.")
  462. while not self.m_stop_event.is_set():
  463. try:
  464. if self.m_cfg.enable and self._is_within_active_hours():
  465. self._poll_booking_signals()
  466. except Exception as exc:
  467. self._log(f"Booking trigger loop exception: {exc}")
  468. self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval)
  469. def _execute_book_job(self, task: Task, query_result: QueryResult):
  470. raise NotImplementedError
  471. def _creator_loop(self):
  472. raise NotImplementedError
  473. def _push_success_notification(self, task_id, order_id, book_res):
  474. """Shared helper to sync booking success state to cloud and dispatch notifications."""
  475. grab_info = {
  476. "order_id": str(order_id) if order_id else None,
  477. "account": book_res.account,
  478. "session_id": book_res.session_id,
  479. "urn": book_res.urn,
  480. "slot_date": book_res.book_date,
  481. "slot_time": book_res.book_time,
  482. "timestamp": int(time.time()),
  483. "payment_link": book_res.payment_link
  484. }
  485. payload = {"status": "grabbed", "grabbed_history": grab_info}
  486. # Commit locally before retiring the browser or dropping its lease.
  487. with self.m_lock:
  488. self._pending_results[str(task_id)] = payload
  489. self._sync_successes()
  490. try:
  491. notify_book_result(book_res, order_id=str(order_id) if order_id else None,
  492. logger=self._log, event_id='booking:order:' + str(task_id))
  493. except Exception as exc:
  494. self._log(f"Booking succeeded, but notification could not be queued: {exc}")
  495. class BuiltinBookerGCO(BaseBookerGCO):
  496. """
  497. Unbound mode (shared built-in account pool):
  498. - Maintains target_instances count of instances globally.
  499. - Instances stay warm; when a slot signal is detected, orders are popped on-demand from the cloud queue.
  500. """
  501. TAG = "BUILTIN-BOOKER"
  502. def _creator_loop(self):
  503. self._log("Creator loop started.")
  504. while not self.m_stop_event.wait(1.0):
  505. if not self._is_within_active_hours():
  506. continue
  507. if time.time() < self.m_spawn_cooldown_until:
  508. continue
  509. with self.m_lock:
  510. current = len(self.m_tasks)
  511. target = self.m_cfg.booker.target_instances
  512. if current < target:
  513. self._spawn_worker()
  514. def _spawn_worker(self):
  515. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  516. return
  517. instance = None
  518. task = None
  519. success = False
  520. plg_cfg = None
  521. try:
  522. plg_cfg = PluginConfig()
  523. plg_cfg.debug = self.m_cfg.debug
  524. plg_cfg.free_config = self.m_cfg.free_config
  525. plg_cfg.session_max_life = self.m_cfg.session_max_life
  526. if self.m_cfg.need_account:
  527. acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  528. plg_cfg.account = type(plg_cfg.account)(**acc)
  529. if self.m_cfg.need_proxy:
  530. proxy = ProxyManager.Instance().get_next_proxy(
  531. self.m_cfg.proxy_pool,
  532. self.m_cfg.proxy_cd,
  533. plugin_name=self.m_cfg.plugin_config.plugin_name
  534. )
  535. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  536. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  537. self._resources.track(instance)
  538. with self._resources.operation(instance):
  539. instance.set_log(self.m_logger)
  540. instance.set_config(plg_cfg)
  541. instance.create_session()
  542. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  543. task = Task(
  544. instance=instance,
  545. next_run=time.time(),
  546. task_ref=None,
  547. acceptable_routing_keys=all_keys,
  548. source_queue="built-in",
  549. book_allowed=True,
  550. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  551. )
  552. with self.m_lock:
  553. if self.m_stop_event.is_set():
  554. return
  555. self.m_tasks.append(task)
  556. success = True
  557. if plg_cfg.proxy.port > 0:
  558. ProxyManager.Instance().mark_proxy_result(
  559. proxy=plg_cfg.proxy.model_dump(),
  560. plugin_name=self.m_cfg.plugin_config.plugin_name,
  561. action="spawn",
  562. success=True,
  563. http_code=200
  564. )
  565. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  566. except Exception as e:
  567. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  568. finally:
  569. if not success and instance:
  570. self._cleanup_instance(instance, "spawn failed")
  571. def _execute_book_job(self, task: Task, query_result: QueryResult):
  572. queue_name = f"auto.{query_result.apt_type.routing_key}"
  573. task_data = None
  574. try:
  575. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  576. except BizLogicError:
  577. # Queue has no pending tasks, return normally
  578. return
  579. except Exception as e:
  580. self._log(f"⚠️ [CloudApi] Failed to pop task from {queue_name}: {e}")
  581. return
  582. if not task_data:
  583. return
  584. task_id = task_data.get('id')
  585. order_id = task_data.get('order_id')
  586. self._lease_order(task_id)
  587. book_res = None
  588. try:
  589. user_input = task_data.get('user_inputs', {})
  590. book_res = task.instance.book(query_result, deepcopy(user_input))
  591. if book_res.success:
  592. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  593. self._push_success_notification(task_id, order_id, book_res)
  594. task.successful_bookings += 1
  595. max_b = self.m_cfg.booker.max_bookings_per_account
  596. if max_b > 0 and task.successful_bookings >= max_b:
  597. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  598. self._remove_task(task, "max bookings reached")
  599. else:
  600. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  601. except Exception as e:
  602. self._dispatch_error_policy("book", e, task=task, task_id=task_id)
  603. finally:
  604. with self.m_lock:
  605. if str(task_id) not in self._pending_results:
  606. self._leased_orders.discard(str(task_id))
  607. if (book_res is None or not book_res.success):
  608. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()})
  609. class OrderBookerGCO(BaseBookerGCO):
  610. """
  611. Bound mode (order provides its own account credentials):
  612. - Maintains warm-up quotas per routing queue.
  613. - Strict 1:1 binding: one instance per cloud order.
  614. - Instance is destroyed immediately after booking completes.
  615. """
  616. TAG = "ORDER-BOOKER"
  617. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  618. super().__init__(cfg, redis_conf, logger)
  619. self.m_task_data_cache: Dict[str, dict] = {}
  620. def _start_additional_threads(self):
  621. thread = threading.Thread(target=self._cache_refresh_loop, daemon=True)
  622. self._background_threads.append(thread)
  623. thread.start()
  624. def _on_task_removed(self, task: Task):
  625. task_id = task.task_ref
  626. if task_id is not None:
  627. self.m_task_data_cache.pop(str(task_id), None)
  628. def _on_all_tasks_cleaned(self):
  629. self.m_task_data_cache.clear()
  630. def _cache_refresh_loop(self):
  631. self._log("Cache refresh loop started.")
  632. refresh_interval = 15 * 60
  633. while not self.m_stop_event.is_set():
  634. try:
  635. if self.m_stop_event.wait(1):
  636. break
  637. with self.m_lock:
  638. tasks_to_check = {
  639. tid: data.get('_last_refresh', 0)
  640. for tid, data in self.m_task_data_cache.items()
  641. }
  642. if not tasks_to_check:
  643. continue
  644. now = time.time()
  645. for tid, last_refresh in tasks_to_check.items():
  646. if self.m_stop_event.is_set():
  647. break
  648. if now - last_refresh >= refresh_interval:
  649. fresh_data = CloudApi.Instance().get_vas_task(tid)
  650. if fresh_data:
  651. fresh_data['_last_refresh'] = time.time()
  652. with self.m_lock:
  653. if tid in self.m_task_data_cache:
  654. self.m_task_data_cache[tid] = fresh_data
  655. self.m_stop_event.wait(0.5)
  656. except Exception as e:
  657. self._log(f'Cache refresh loop exception: {e}')
  658. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  659. if query_result.availability_status != AvailabilityStatus.Available:
  660. return True
  661. task_id = task.task_ref
  662. task_data = self.m_task_data_cache.get(str(task_id), {})
  663. user_input = task_data.get('user_inputs', {})
  664. expected_end_date = (
  665. user_input.get('expected_end_date')
  666. or '2100-01-01'
  667. )
  668. available_date = query_result.earliest_date
  669. if available_date is None:
  670. return False
  671. dt = available_date.strftime("%Y-%m-%d")
  672. return dt <= expected_end_date
  673. def _execute_book_job(self, task: Task, query_result: QueryResult):
  674. task_id = task.task_ref
  675. task_data = None
  676. try:
  677. with self.m_lock:
  678. task_data = self.m_task_data_cache.get(str(task_id))
  679. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']:
  680. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  681. self._remove_task(task, "bound task no longer valid")
  682. self.redis_client.zrem(self.m_tracker_key, task_id)
  683. return
  684. order_id = task_data.get('order_id')
  685. user_input = task_data.get('user_inputs', {})
  686. book_res = task.instance.book(query_result, deepcopy(user_input))
  687. if book_res.success:
  688. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  689. self._push_success_notification(task_id, order_id, book_res)
  690. self._remove_task(task, "booking success")
  691. else:
  692. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  693. except Exception as e:
  694. self._dispatch_error_policy("book", e, task=task)
  695. def _creator_loop(self):
  696. self._log("Creator loop started.")
  697. while not self.m_stop_event.wait(1.0):
  698. if not self._is_within_active_hours():
  699. continue
  700. if time.time() < self.m_spawn_cooldown_until:
  701. continue
  702. for apt in self.m_cfg.appointment_types:
  703. r_key = apt.routing_key
  704. with self.m_lock:
  705. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  706. target = self.m_cfg.booker.target_instances
  707. if active < target:
  708. self._spawn_worker(r_key)
  709. def _spawn_worker(self, target_routing_key: str):
  710. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  711. return
  712. instance = None
  713. task = None
  714. success = False
  715. task_id = None
  716. plg_cfg = None
  717. try:
  718. queue_name = f"auto.{target_routing_key}"
  719. try:
  720. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  721. except BizLogicError:
  722. # Queue has no pending tasks, return directly
  723. return
  724. if not task_data:
  725. return
  726. task_id = task_data['id']
  727. with self.m_lock:
  728. self.m_task_data_cache[str(task_id)] = task_data
  729. self._lease_order(task_id)
  730. user_inputs = task_data.get('user_inputs', {})
  731. plg_cfg = PluginConfig()
  732. plg_cfg.debug = self.m_cfg.debug
  733. plg_cfg.free_config = self.m_cfg.free_config
  734. plg_cfg.session_max_life = self.m_cfg.session_max_life
  735. plg_cfg.account.username = user_inputs.get("username", "")
  736. plg_cfg.account.password = user_inputs.get("password", "")
  737. if not plg_cfg.account.username:
  738. raise AccountIncompleteError(f"Task={task_id} missing username credentials in user_inputs")
  739. acceptable_keys = [target_routing_key]
  740. if self.m_cfg.need_proxy:
  741. proxy = ProxyManager.Instance().get_next_proxy(
  742. self.m_cfg.proxy_pool,
  743. self.m_cfg.proxy_cd,
  744. plugin_name=self.m_cfg.plugin_config.plugin_name
  745. )
  746. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  747. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  748. self._resources.track(instance)
  749. with self._resources.operation(instance):
  750. instance.set_log(self.m_logger)
  751. instance.set_config(plg_cfg)
  752. instance.create_session()
  753. task = Task(
  754. instance=instance,
  755. next_run=time.time(),
  756. task_ref=task_id,
  757. acceptable_routing_keys=acceptable_keys,
  758. source_queue=target_routing_key,
  759. book_allowed=True,
  760. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  761. )
  762. with self.m_lock:
  763. if self.m_stop_event.is_set():
  764. return
  765. self.m_tasks.append(task)
  766. success = True
  767. if plg_cfg.proxy.port > 0:
  768. ProxyManager.Instance().mark_proxy_result(
  769. proxy=plg_cfg.proxy.model_dump(),
  770. plugin_name=self.m_cfg.plugin_config.plugin_name,
  771. action="spawn",
  772. success=True,
  773. http_code=200
  774. )
  775. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  776. except Exception as e:
  777. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
  778. finally:
  779. if not success and instance:
  780. self._cleanup_instance(instance, "spawn failed")
  781. if not success and task_id is not None:
  782. with self.m_lock:
  783. self.m_task_data_cache.pop(str(task_id), None)
  784. try:
  785. with self.m_lock:
  786. self._leased_orders.discard(str(task_id))
  787. # Retain a recovery deadline (including any 429 cooldown).
  788. self.redis_client.zadd(self.m_tracker_key, {
  789. str(task_id): max(time.time(), self.m_spawn_cooldown_until)})
  790. except Exception:
  791. pass