booker.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891
  1. import os
  2. from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
  3. import time
  4. import json
  5. import threading
  6. import random
  7. from datetime import datetime, timedelta, timezone
  8. from typing import List, Dict, Callable, Optional
  9. from core_types import (
  10. GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult,
  11. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
  12. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  13. NetworkTimeoutError, BizLogicError, AccountStateError, AccountAuthError,
  14. AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error
  15. )
  16. from plugin_factory import PluginFactory
  17. from toolkit.cloud_api import CloudApi
  18. from toolkit.proxy_manager import ProxyManager
  19. from toolkit.task_executor import TaskExecutor
  20. from utils.booking_outbox import BookingOutbox
  21. from toolkit.notifier import notify_book_result
  22. from utils.safe_redis_cli import SafeRedisClient
  23. class BaseBookerGCO:
  24. """
  25. Base Booker class encapsulating shared infrastructure, lifecycle management, and logic loops.
  26. """
  27. TAG = "BOOKER"
  28. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  29. self.m_cfg = cfg
  30. self.m_factory = PluginFactory()
  31. self.m_logger = logger
  32. self.m_tasks: List[Task] = []
  33. self.m_lock = threading.RLock()
  34. self.m_stop_event = threading.Event()
  35. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  36. init_lifecycle(self)
  37. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  38. self.m_last_group_query_time = 0.0
  39. self.m_spawn_cooldown_until = 0.0
  40. self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking")
  41. self._leased_orders = set()
  42. self._pending_results = {}
  43. self._outbox = BookingOutbox()
  44. self._background_threads = []
  45. self._heartbeat_stop_event = threading.Event()
  46. def _log(self, message):
  47. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  48. if self.m_logger:
  49. self.m_logger(f'{prefix} {message}')
  50. else:
  51. print(f'{prefix} {message}')
  52. def get_status(self) -> GroupStatus:
  53. with self.m_lock:
  54. tasks = list(self.m_tasks)
  55. gid = self.m_cfg.identifier
  56. enable = self.m_cfg.enable
  57. plugin_name = self.m_cfg.plugin_config.plugin_name
  58. plugins_status = []
  59. healthy_count = 0
  60. for t in tasks:
  61. if t and t.instance:
  62. st = t.instance.get_plugin_status()
  63. if st.is_healthy:
  64. healthy_count += 1
  65. plugins_status.append(st)
  66. return GroupStatus(
  67. group_id=gid,
  68. enable=enable,
  69. plugin_name=plugin_name,
  70. total_tasks=len(tasks),
  71. healthy_tasks=healthy_count,
  72. plugins=plugins_status,
  73. updated_at=utc_now_str(),
  74. )
  75. @start_once
  76. def start(self):
  77. if not self.m_cfg.enable:
  78. return
  79. self._log(f"Starting {self.TAG}...")
  80. plugin_name = self.m_cfg.plugin_config.plugin_name
  81. class_name = "".join(part.title() for part in plugin_name.split('_'))
  82. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  83. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  84. for target in (self._booking_trigger_loop, self._creator_loop,
  85. self._maintain_loop, self._heartbeat_loop, self._success_sync_loop):
  86. thread = threading.Thread(target=target, daemon=True, name=target.__name__)
  87. self._background_threads.append(thread)
  88. thread.start()
  89. self._start_additional_threads()
  90. def _start_additional_threads(self):
  91. """Hook for subclasses to start additional background threads."""
  92. pass
  93. @serialized_stop
  94. def stop(self):
  95. self._log("Stopping Booker...")
  96. self.m_stop_event.set()
  97. deadline = time.monotonic() + self.m_cfg.shutdown_timeout
  98. self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
  99. workers = [t for t in self._background_threads if t.name != '_heartbeat_loop']
  100. join_threads(workers, max(0, deadline - time.monotonic()))
  101. with self.m_lock:
  102. for task in self.m_tasks:
  103. task.removed = True
  104. self._resources.track(task.instance)
  105. self._resources.close(self.m_cfg.cleanup_timeout)
  106. threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1)
  107. if threads_done and not self._pending_results:
  108. self._heartbeat_stop_event.set()
  109. threads_done = join_threads(
  110. [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2)
  111. if not self._resources.empty or not threads_done or self._pending_results:
  112. raise RuntimeError('Booker shutdown incomplete; resources retained for retry')
  113. with self.m_lock:
  114. self.m_tasks.clear()
  115. self._on_all_tasks_cleaned()
  116. self._leased_orders.clear()
  117. self.redis_client.close()
  118. def update_config(self, new_cfg: GroupConfig):
  119. """Dynamically updates the configuration."""
  120. with self.m_lock:
  121. if self.m_cfg.enable and not new_cfg.enable:
  122. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  123. elif not self.m_cfg.enable and new_cfg.enable:
  124. self._log("Config dynamically updated: Group ENABLED.")
  125. else:
  126. self._log("Config dynamically updated: Parameters refreshed.")
  127. self.m_cfg = new_cfg
  128. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  129. if instance is None:
  130. return True
  131. cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
  132. if not cleaned:
  133. self._log(f"Cleanup retained for retry ({reason})")
  134. return cleaned
  135. def _cleanup_task(self, task: Task, reason: str = ""):
  136. if task and task.instance:
  137. with task._operation_lock:
  138. self._cleanup_instance(task.instance, reason, task.task_ref)
  139. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  140. removed = False
  141. with self.m_lock:
  142. if task in self.m_tasks:
  143. task.removed = True
  144. self.m_tasks.remove(task)
  145. removed = True
  146. if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
  147. self._leased_orders.discard(str(task.task_ref))
  148. self._on_task_removed(task)
  149. if cleanup and removed:
  150. self._cleanup_task(task, reason)
  151. return removed
  152. def _on_task_removed(self, task: Task):
  153. """Hook for subclasses when a task is removed (e.g. cleaning up caches)."""
  154. pass
  155. def _cleanup_all_tasks(self, reason: str = ""):
  156. with self.m_lock:
  157. tasks = list(self.m_tasks)
  158. self.m_tasks.clear()
  159. for task in tasks:
  160. task.removed = True
  161. if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
  162. self._leased_orders.discard(str(task.task_ref))
  163. self._on_all_tasks_cleaned()
  164. for task in tasks:
  165. self._cleanup_task(task, reason)
  166. def _on_all_tasks_cleaned(self):
  167. """Hook for subclasses when all tasks are cleared."""
  168. pass
  169. def _get_redis_key(self, routing_key: str) -> str:
  170. return f"vs:signal:{routing_key}"
  171. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  172. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  173. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  174. try:
  175. CloudApi.Instance().lock_account(acc.id, 0)
  176. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  177. except Exception as e:
  178. self._log(f"Failed to release account [{acc.username}]: {e}")
  179. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  180. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  181. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  182. return
  183. try:
  184. cloud = CloudApi.Instance()
  185. url = f"{cloud.base_url}/api/account/update"
  186. params = {"account_id": acc.id}
  187. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  188. payload = {
  189. "status": "disable",
  190. "next_use_time": far_future,
  191. "extra_data": {
  192. "disable_reason": reason,
  193. "disabled_at": int(time.time()),
  194. "module": self.TAG,
  195. "group": getattr(self.m_cfg, "identifier", "")
  196. }
  197. }
  198. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  199. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  200. except Exception as e:
  201. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  202. try:
  203. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  204. except Exception as lock_err:
  205. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  206. def _dispatch_error_policy(
  207. self,
  208. action: str,
  209. err: Exception,
  210. task: Optional[Task] = None,
  211. plg_cfg: Optional[PluginConfig] = None,
  212. task_id: Optional[int] = None
  213. ) -> None:
  214. """
  215. Booker error dispatch and self-healing policy engine:
  216. Executes self-healing actions based on the (action, error type) matrix:
  217. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account or fails order task, never releases back to pool.
  218. - 429 during spawn/login: Account-level rate limit. Locks account, updates tracker & cooldown.
  219. - 429 during book: Booking slot rate limit or egress IP throttle. Preserves account, removes task to rotate proxy.
  220. - 401 / session expired: Session invalidated. Removes task.
  221. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  222. - Handles logging and timer backoffs internally.
  223. """
  224. biz_err = normalize_error(err)
  225. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  226. acc = active_plg_cfg.account if active_plg_cfg else None
  227. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  228. if isinstance(biz_err, AccountStateError):
  229. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  230. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  231. # If bound order task, update VAS task status to failed
  232. if effective_task_id is not None:
  233. try:
  234. CloudApi.Instance().update_vas_task(str(effective_task_id), {
  235. "status": "failed",
  236. "fail_reason": f"Account fatal error: {biz_err.message} (code={biz_err.code})"
  237. })
  238. self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
  239. except Exception as update_err:
  240. self._log(f"Failed to update VAS task status: {update_err}")
  241. else:
  242. self.redis_client.zrem(self.m_tracker_key, effective_task_id)
  243. # If pool account, permanently disable it in cloud
  244. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  245. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  246. # Never release account back to pool
  247. if task:
  248. self._remove_task(task, f"account fatal error: {biz_err.message}")
  249. if action in ("spawn", "login", "create_session"):
  250. self.m_spawn_cooldown_until = time.time() + 5.0
  251. # 1. Target site rate limit / 429
  252. elif isinstance(biz_err, RateLimiteddError):
  253. if action in ("spawn", "login", "create_session"):
  254. backoff = float(self.m_cfg.login_backoff)
  255. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  256. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  257. try:
  258. CloudApi.Instance().lock_account(acc.id, int(backoff))
  259. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  260. except Exception as lock_err:
  261. self._log(f"Failed to lock account via cloud API: {lock_err}")
  262. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  263. if effective_task_id is not None:
  264. self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
  265. self.m_spawn_cooldown_until = time.time() + backoff
  266. if task:
  267. self._remove_task(task, "spawn account rate limited")
  268. elif action == "book":
  269. # 429 during booking: slot rate limit or IP restriction, preserve account and remove task to rotate proxy
  270. self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
  271. if task:
  272. self._remove_task(task, "booking rate limited")
  273. else:
  274. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  275. if task:
  276. self._remove_task(task, f"{action} rate limited")
  277. # 2. Session expired / 401
  278. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  279. if action in ("spawn", "login", "create_session"):
  280. self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
  281. self.m_spawn_cooldown_until = time.time() + 5.0
  282. else:
  283. self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
  284. if task:
  285. self._remove_task(task, f"{action} session expired")
  286. # 3. Permission denied / WAF blocked / 403
  287. elif isinstance(biz_err, PermissionDeniedError):
  288. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  289. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  290. if task:
  291. self._remove_task(task, f"{action} waf blocked")
  292. if action in ("spawn", "login", "create_session"):
  293. self._release_account_safely(acc, reason="WAF Blocked")
  294. self.m_spawn_cooldown_until = time.time() + proxy_cd
  295. # 4. Network or browser timeout
  296. elif isinstance(biz_err, NetworkTimeoutError):
  297. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  298. if task:
  299. task.next_run = time.time() + 5.0
  300. if action in ("spawn", "login", "create_session"):
  301. self._release_account_safely(acc, reason="Network Timeout")
  302. spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
  303. self.m_spawn_cooldown_until = time.time() + spawn_cd
  304. # 5. Fallback policy
  305. else:
  306. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  307. if action in ("spawn", "login", "create_session"):
  308. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  309. spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
  310. self.m_spawn_cooldown_until = time.time() + spawn_cd
  311. elif task:
  312. task.next_run = time.time() + 10.0
  313. # 6. Proxy health metrics recording
  314. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  315. ProxyManager.Instance().mark_proxy_result(
  316. proxy=active_plg_cfg.proxy.model_dump(),
  317. plugin_name=self.m_cfg.plugin_config.plugin_name,
  318. action=action,
  319. success=False,
  320. http_code=biz_err.http_status,
  321. error_type=biz_err.message
  322. )
  323. def _is_within_active_hours(self) -> bool:
  324. """Checks if current time is within allowed operating hours (UTC)."""
  325. start_str = self.m_cfg.active_time_start
  326. end_str = self.m_cfg.active_time_end
  327. if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
  328. return True
  329. current_utc_time = datetime.now(timezone.utc).time()
  330. start_time = datetime.strptime(start_str, "%H:%M").time()
  331. end_time = datetime.strptime(end_str, "%H:%M").time()
  332. if start_time <= end_time:
  333. return start_time <= current_utc_time <= end_time
  334. else:
  335. return current_utc_time >= start_time or current_utc_time <= end_time
  336. def _lease_order(self, task_id):
  337. with self.m_lock:
  338. self._leased_orders.add(str(task_id))
  339. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120})
  340. def _renew_leases(self):
  341. with self.m_lock:
  342. if self._leased_orders:
  343. self.redis_client.bulk_zadd(self.m_tracker_key, {
  344. tid: time.time() + 120 for tid in self._leased_orders})
  345. def _heartbeat_loop(self):
  346. # No browser calls here: long OTP/HTTP waits must not stop order leases.
  347. while not self._heartbeat_stop_event.wait(10):
  348. self._renew_leases()
  349. def _sync_successes(self):
  350. try:
  351. with self.m_lock:
  352. pending = dict(self._pending_results)
  353. for task_id, payload in pending.items():
  354. saved = False
  355. try:
  356. self._outbox.put(task_id, self.m_tracker_key, payload)
  357. saved = True
  358. except Exception as exc:
  359. self._log(f"Local success journal unavailable for task={task_id}: {exc}")
  360. saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved
  361. if saved:
  362. with self.m_lock:
  363. self._pending_results.pop(task_id, None)
  364. self._leased_orders.discard(task_id)
  365. self._outbox.flush(CloudApi.Instance(), self.redis_client._client,
  366. self._log, tracker=self.m_tracker_key)
  367. except Exception as exc:
  368. self._log(f"Success outbox retry failed: {exc}")
  369. def _success_sync_loop(self):
  370. while not self.m_stop_event.is_set() or self._pending_results:
  371. self._sync_successes()
  372. if self.m_stop_event.is_set() and not self._pending_results:
  373. break
  374. if self.m_stop_event.is_set():
  375. self._heartbeat_stop_event.wait(5)
  376. else:
  377. self.m_stop_event.wait(5)
  378. def _maintain_task(self, task):
  379. try:
  380. with task._operation_lock, self._resources.operation(task.instance):
  381. if task.removed or self.m_stop_event.is_set():
  382. return
  383. task.instance.keep_alive()
  384. if task.instance.health_check():
  385. task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5))
  386. else:
  387. self._remove_task(task, "unhealthy or keep-alive failed")
  388. except Exception as exc:
  389. task.next_remote_ping = time.time() + 5
  390. self._dispatch_error_policy("keep_alive", exc, task=task)
  391. finally:
  392. with self.m_lock:
  393. task.busy = False
  394. def _schedule_maintenance(self):
  395. with self.m_lock:
  396. if self.m_stop_event.is_set():
  397. return
  398. for task in self.m_tasks:
  399. if task.busy or task.removed or time.time() < task.next_remote_ping:
  400. continue
  401. task.busy = True
  402. if not self._executor.submit(self._maintain_task, task):
  403. task.busy = False
  404. def _maintain_loop(self):
  405. while not self.m_stop_event.wait(1):
  406. self._schedule_maintenance()
  407. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  408. """Checks if available dates in query_result match task preferences (defaults to True)."""
  409. return True
  410. def _run_book_job(self, task, query_result):
  411. try:
  412. with task._operation_lock, self._resources.operation(task.instance):
  413. if not task.removed and not self.m_stop_event.is_set():
  414. self._execute_book_job(task, query_result)
  415. finally:
  416. with self.m_lock:
  417. task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown)
  418. task.busy = False
  419. def _poll_booking_signals(self):
  420. now = time.time()
  421. with self.m_lock:
  422. ready = [t for t in self.m_tasks if not t.busy and not t.removed
  423. and t.book_allowed and now >= t.next_run]
  424. keys = {key for t in ready for key in t.acceptable_routing_keys}
  425. apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys]
  426. if not apt_types:
  427. return
  428. redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types]
  429. signals = self.redis_client.mget(redis_keys)
  430. for apt_type, raw_data in zip(apt_types, signals):
  431. if not raw_data:
  432. continue
  433. try:
  434. data = json.loads(raw_data)
  435. result = QueryResult.model_validate(data['query_result'])
  436. result.apt_type = AppointmentType.model_validate(data['apt_type'])
  437. if result.apt_type.routing_key != apt_type.routing_key:
  438. raise ValueError('signal routing key mismatch')
  439. except Exception as exc:
  440. # A producer may have replaced this value since MGET. Do not
  441. # delete a newer signal based on an old malformed payload.
  442. self._log(f"Invalid signal for {apt_type.routing_key}: {exc}")
  443. continue
  444. with self.m_lock:
  445. for task in ready:
  446. if (task.busy or task.removed or self.m_stop_event.is_set()
  447. or apt_type.routing_key not in task.acceptable_routing_keys
  448. or not self._is_date_of_interest(task, result)):
  449. continue
  450. task.busy = True
  451. if not self._executor.submit(self._run_book_job, task, result):
  452. task.busy = False
  453. def _booking_trigger_loop(self):
  454. self._log("Trigger loop started.")
  455. while not self.m_stop_event.is_set():
  456. try:
  457. if self.m_cfg.enable and self._is_within_active_hours():
  458. self._poll_booking_signals()
  459. except Exception as exc:
  460. self._log(f"Booking trigger loop exception: {exc}")
  461. self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval)
  462. def _execute_book_job(self, task: Task, query_result: QueryResult):
  463. raise NotImplementedError
  464. def _creator_loop(self):
  465. raise NotImplementedError
  466. def _push_success_notification(self, task_id, order_id, book_res):
  467. """Shared helper to sync booking success state to cloud and dispatch notifications."""
  468. grab_info = {
  469. "account": book_res.account,
  470. "session_id": book_res.session_id,
  471. "urn": book_res.urn,
  472. "slot_date": book_res.book_date,
  473. "slot_time": book_res.book_time,
  474. "timestamp": int(time.time()),
  475. "payment_link": book_res.payment_link
  476. }
  477. payload = {"status": "grabbed", "grabbed_history": grab_info}
  478. # Commit locally before retiring the browser or dropping its lease.
  479. with self.m_lock:
  480. self._pending_results[str(task_id)] = payload
  481. self._sync_successes()
  482. try:
  483. notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
  484. except Exception as exc:
  485. self._log(f"Booking succeeded, but notification could not be queued: {exc}")
  486. class BuiltinBookerGCO(BaseBookerGCO):
  487. """
  488. Unbound mode (shared built-in account pool):
  489. - Maintains target_instances count of instances globally.
  490. - Instances stay warm; when a slot signal is detected, orders are popped on-demand from the cloud queue.
  491. """
  492. TAG = "BUILTIN-BOOKER"
  493. def _creator_loop(self):
  494. self._log("Creator loop started.")
  495. while not self.m_stop_event.wait(1.0):
  496. if not self._is_within_active_hours():
  497. continue
  498. if time.time() < self.m_spawn_cooldown_until:
  499. continue
  500. with self.m_lock:
  501. current = len(self.m_tasks)
  502. target = self.m_cfg.booker.target_instances
  503. if current < target:
  504. self._spawn_worker()
  505. def _spawn_worker(self):
  506. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  507. return
  508. instance = None
  509. task = None
  510. success = False
  511. plg_cfg = None
  512. try:
  513. plg_cfg = PluginConfig()
  514. plg_cfg.debug = self.m_cfg.debug
  515. plg_cfg.free_config = self.m_cfg.free_config
  516. plg_cfg.session_max_life = self.m_cfg.session_max_life
  517. if self.m_cfg.need_account:
  518. acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  519. plg_cfg.account = type(plg_cfg.account)(**acc)
  520. if self.m_cfg.need_proxy:
  521. proxy = ProxyManager.Instance().get_next_proxy(
  522. self.m_cfg.proxy_pool,
  523. self.m_cfg.proxy_cd,
  524. plugin_name=self.m_cfg.plugin_config.plugin_name
  525. )
  526. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  527. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  528. self._resources.track(instance)
  529. with self._resources.operation(instance):
  530. instance.set_log(self.m_logger)
  531. instance.set_config(plg_cfg)
  532. instance.create_session()
  533. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  534. task = Task(
  535. instance=instance,
  536. next_run=time.time(),
  537. task_ref=None,
  538. acceptable_routing_keys=all_keys,
  539. source_queue="built-in",
  540. book_allowed=True,
  541. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  542. )
  543. with self.m_lock:
  544. if self.m_stop_event.is_set():
  545. return
  546. self.m_tasks.append(task)
  547. success = True
  548. if plg_cfg.proxy.port > 0:
  549. ProxyManager.Instance().mark_proxy_result(
  550. proxy=plg_cfg.proxy.model_dump(),
  551. plugin_name=self.m_cfg.plugin_config.plugin_name,
  552. action="spawn",
  553. success=True,
  554. http_code=200
  555. )
  556. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  557. except Exception as e:
  558. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  559. finally:
  560. if not success and instance:
  561. self._cleanup_instance(instance, "spawn failed")
  562. def _execute_book_job(self, task: Task, query_result: QueryResult):
  563. queue_name = f"auto.{query_result.apt_type.routing_key}"
  564. task_data = None
  565. try:
  566. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  567. except BizLogicError:
  568. # Queue has no pending tasks, return normally
  569. return
  570. except Exception as e:
  571. self._log(f"⚠️ [CloudApi] Failed to pop task from {queue_name}: {e}")
  572. return
  573. if not task_data:
  574. return
  575. task_id = task_data.get('id')
  576. order_id = task_data.get('order_id')
  577. self._lease_order(task_id)
  578. book_res = None
  579. try:
  580. user_input = task_data.get('user_inputs', {})
  581. book_res = task.instance.book(query_result, user_input)
  582. if book_res.success:
  583. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  584. self._push_success_notification(task_id, order_id, book_res)
  585. task.successful_bookings += 1
  586. max_b = self.m_cfg.booker.max_bookings_per_account
  587. if max_b > 0 and task.successful_bookings >= max_b:
  588. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  589. self._remove_task(task, "max bookings reached")
  590. else:
  591. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  592. except Exception as e:
  593. self._dispatch_error_policy("book", e, task=task, task_id=task_id)
  594. finally:
  595. with self.m_lock:
  596. if str(task_id) not in self._pending_results:
  597. self._leased_orders.discard(str(task_id))
  598. if book_res is None or not book_res.success:
  599. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()})
  600. class OrderBookerGCO(BaseBookerGCO):
  601. """
  602. Bound mode (order provides its own account credentials):
  603. - Maintains warm-up quotas per routing queue.
  604. - Strict 1:1 binding: one instance per cloud order.
  605. - Instance is destroyed immediately after booking completes.
  606. """
  607. TAG = "ORDER-BOOKER"
  608. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  609. super().__init__(cfg, redis_conf, logger)
  610. self.m_task_data_cache: Dict[str, dict] = {}
  611. def _start_additional_threads(self):
  612. thread = threading.Thread(target=self._cache_refresh_loop, daemon=True)
  613. self._background_threads.append(thread)
  614. thread.start()
  615. def _on_task_removed(self, task: Task):
  616. task_id = task.task_ref
  617. if task_id is not None:
  618. self.m_task_data_cache.pop(str(task_id), None)
  619. def _on_all_tasks_cleaned(self):
  620. self.m_task_data_cache.clear()
  621. def _cache_refresh_loop(self):
  622. self._log("Cache refresh loop started.")
  623. refresh_interval = 15 * 60
  624. while not self.m_stop_event.is_set():
  625. try:
  626. if self.m_stop_event.wait(1):
  627. break
  628. with self.m_lock:
  629. tasks_to_check = {
  630. tid: data.get('_last_refresh', 0)
  631. for tid, data in self.m_task_data_cache.items()
  632. }
  633. if not tasks_to_check:
  634. continue
  635. now = time.time()
  636. for tid, last_refresh in tasks_to_check.items():
  637. if self.m_stop_event.is_set():
  638. break
  639. if now - last_refresh >= refresh_interval:
  640. fresh_data = CloudApi.Instance().get_vas_task(tid)
  641. if fresh_data:
  642. fresh_data['_last_refresh'] = time.time()
  643. with self.m_lock:
  644. if tid in self.m_task_data_cache:
  645. self.m_task_data_cache[tid] = fresh_data
  646. self.m_stop_event.wait(0.5)
  647. except Exception as e:
  648. self._log(f'Cache refresh loop exception: {e}')
  649. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  650. if query_result.availability_status != AvailabilityStatus.Available:
  651. return True
  652. task_id = task.task_ref
  653. task_data = self.m_task_data_cache.get(str(task_id), {})
  654. user_input = task_data.get('user_inputs', {})
  655. expected_end_date = (
  656. user_input.get('expected_end_date')
  657. or '2100-01-01'
  658. )
  659. available_date = query_result.earliest_date
  660. if available_date is None:
  661. return False
  662. dt = available_date.strftime("%Y-%m-%d")
  663. return dt <= expected_end_date
  664. def _execute_book_job(self, task: Task, query_result: QueryResult):
  665. task_id = task.task_ref
  666. task_data = None
  667. try:
  668. with self.m_lock:
  669. task_data = self.m_task_data_cache.get(str(task_id))
  670. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']:
  671. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  672. self._remove_task(task, "bound task no longer valid")
  673. self.redis_client.zrem(self.m_tracker_key, task_id)
  674. return
  675. order_id = task_data.get('order_id')
  676. user_input = task_data.get('user_inputs', {})
  677. book_res = task.instance.book(query_result, user_input)
  678. if book_res.success:
  679. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  680. self._push_success_notification(task_id, order_id, book_res)
  681. self._remove_task(task, "booking success")
  682. else:
  683. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  684. except Exception as e:
  685. self._dispatch_error_policy("book", e, task=task)
  686. def _creator_loop(self):
  687. self._log("Creator loop started.")
  688. while not self.m_stop_event.wait(1.0):
  689. if not self._is_within_active_hours():
  690. continue
  691. if time.time() < self.m_spawn_cooldown_until:
  692. continue
  693. for apt in self.m_cfg.appointment_types:
  694. r_key = apt.routing_key
  695. with self.m_lock:
  696. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  697. target = self.m_cfg.booker.target_instances
  698. if active < target:
  699. self._spawn_worker(r_key)
  700. def _spawn_worker(self, target_routing_key: str):
  701. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  702. return
  703. instance = None
  704. task = None
  705. success = False
  706. task_id = None
  707. plg_cfg = None
  708. try:
  709. queue_name = f"auto.{target_routing_key}"
  710. try:
  711. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  712. except BizLogicError:
  713. # Queue has no pending tasks, return directly
  714. return
  715. if not task_data:
  716. return
  717. task_id = task_data['id']
  718. with self.m_lock:
  719. self.m_task_data_cache[str(task_id)] = task_data
  720. self._lease_order(task_id)
  721. user_inputs = task_data.get('user_inputs', {})
  722. plg_cfg = PluginConfig()
  723. plg_cfg.debug = self.m_cfg.debug
  724. plg_cfg.free_config = self.m_cfg.free_config
  725. plg_cfg.session_max_life = self.m_cfg.session_max_life
  726. plg_cfg.account.username = user_inputs.get("username", "")
  727. plg_cfg.account.password = user_inputs.get("password", "")
  728. if not plg_cfg.account.username:
  729. raise AccountIncompleteError(f"Task={task_id} missing username credentials in user_inputs")
  730. acceptable_keys = [target_routing_key]
  731. if self.m_cfg.need_proxy:
  732. proxy = ProxyManager.Instance().get_next_proxy(
  733. self.m_cfg.proxy_pool,
  734. self.m_cfg.proxy_cd,
  735. plugin_name=self.m_cfg.plugin_config.plugin_name
  736. )
  737. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  738. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  739. self._resources.track(instance)
  740. with self._resources.operation(instance):
  741. instance.set_log(self.m_logger)
  742. instance.set_config(plg_cfg)
  743. instance.create_session()
  744. task = Task(
  745. instance=instance,
  746. next_run=time.time(),
  747. task_ref=task_id,
  748. acceptable_routing_keys=acceptable_keys,
  749. source_queue=target_routing_key,
  750. book_allowed=True,
  751. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  752. )
  753. with self.m_lock:
  754. if self.m_stop_event.is_set():
  755. return
  756. self.m_tasks.append(task)
  757. success = True
  758. if plg_cfg.proxy.port > 0:
  759. ProxyManager.Instance().mark_proxy_result(
  760. proxy=plg_cfg.proxy.model_dump(),
  761. plugin_name=self.m_cfg.plugin_config.plugin_name,
  762. action="spawn",
  763. success=True,
  764. http_code=200
  765. )
  766. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  767. except Exception as e:
  768. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
  769. finally:
  770. if not success and instance:
  771. self._cleanup_instance(instance, "spawn failed")
  772. if not success and task_id is not None:
  773. with self.m_lock:
  774. self.m_task_data_cache.pop(str(task_id), None)
  775. try:
  776. with self.m_lock:
  777. self._leased_orders.discard(str(task_id))
  778. # Retain a recovery deadline (including any 429 cooldown).
  779. self.redis_client.zadd(self.m_tracker_key, {
  780. str(task_id): max(time.time(), self.m_spawn_cooldown_until)})
  781. except Exception:
  782. pass