booker.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889
  1. import os
  2. from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
  3. import time
  4. import json
  5. import threading
  6. import random
  7. from datetime import datetime, timedelta, timezone
  8. from typing import List, Dict, Callable, Optional
  9. from core_types import (
  10. GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult,
  11. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
  12. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  13. NetworkTimeoutError, BizLogicError, AccountStateError, AccountAuthError,
  14. AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error
  15. )
  16. from plugin_factory import PluginFactory
  17. from toolkit.cloud_api import CloudApi
  18. from toolkit.proxy_manager import ProxyManager
  19. from toolkit.task_executor import TaskExecutor
  20. from utils.booking_outbox import BookingOutbox
  21. from toolkit.notifier import notify_book_result
  22. from utils.safe_redis_cli import SafeRedisClient
  23. class BaseBookerGCO:
  24. """
  25. Base Booker class encapsulating shared infrastructure, lifecycle management, and logic loops.
  26. """
  27. TAG = "BOOKER"
  28. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  29. self.m_cfg = cfg
  30. self.m_factory = PluginFactory()
  31. self.m_logger = logger
  32. self.m_tasks: List[Task] = []
  33. self.m_lock = threading.RLock()
  34. self.m_stop_event = threading.Event()
  35. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  36. init_lifecycle(self)
  37. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  38. self.m_last_group_query_time = 0.0
  39. self.m_spawn_cooldown_until = 0.0
  40. self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking")
  41. self._leased_orders = set()
  42. self._pending_results = {}
  43. self._outbox = BookingOutbox()
  44. self._background_threads = []
  45. self._heartbeat_stop_event = threading.Event()
  46. def _log(self, message):
  47. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  48. if self.m_logger:
  49. self.m_logger(f'{prefix} {message}')
  50. else:
  51. print(f'{prefix} {message}')
  52. def get_status(self) -> GroupStatus:
  53. with self.m_lock:
  54. tasks = list(self.m_tasks)
  55. gid = self.m_cfg.identifier
  56. enable = self.m_cfg.enable
  57. plugin_name = self.m_cfg.plugin_config.plugin_name
  58. plugins_status = []
  59. healthy_count = 0
  60. for t in tasks:
  61. if t and t.instance:
  62. st = t.instance.get_plugin_status()
  63. if st.is_healthy:
  64. healthy_count += 1
  65. plugins_status.append(st)
  66. return GroupStatus(
  67. group_id=gid,
  68. enable=enable,
  69. plugin_name=plugin_name,
  70. total_tasks=len(tasks),
  71. healthy_tasks=healthy_count,
  72. plugins=plugins_status,
  73. updated_at=utc_now_str(),
  74. )
  75. @start_once
  76. def start(self):
  77. if not self.m_cfg.enable:
  78. return
  79. self._log(f"Starting {self.TAG}...")
  80. plugin_name = self.m_cfg.plugin_config.plugin_name
  81. class_name = "".join(part.title() for part in plugin_name.split('_'))
  82. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  83. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  84. for target in (self._booking_trigger_loop, self._creator_loop,
  85. self._maintain_loop, self._heartbeat_loop, self._success_sync_loop):
  86. thread = threading.Thread(target=target, daemon=True, name=target.__name__)
  87. self._background_threads.append(thread)
  88. thread.start()
  89. self._start_additional_threads()
  90. def _start_additional_threads(self):
  91. """Hook for subclasses to start additional background threads."""
  92. pass
  93. @serialized_stop
  94. def stop(self):
  95. self._log("Stopping Booker...")
  96. self.m_stop_event.set()
  97. deadline = time.monotonic() + self.m_cfg.shutdown_timeout
  98. self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
  99. workers = [t for t in self._background_threads if t.name != '_heartbeat_loop']
  100. join_threads(workers, max(0, deadline - time.monotonic()))
  101. with self.m_lock:
  102. for task in self.m_tasks:
  103. task.removed = True
  104. self._resources.track(task.instance)
  105. self._resources.close(self.m_cfg.cleanup_timeout)
  106. threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1)
  107. if threads_done and not self._pending_results:
  108. self._heartbeat_stop_event.set()
  109. threads_done = join_threads(
  110. [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2)
  111. if not self._resources.empty or not threads_done or self._pending_results:
  112. raise RuntimeError('Booker shutdown incomplete; resources retained for retry')
  113. with self.m_lock:
  114. self.m_tasks.clear()
  115. self._on_all_tasks_cleaned()
  116. self._leased_orders.clear()
  117. self.redis_client.close()
  118. def update_config(self, new_cfg: GroupConfig):
  119. """Dynamically updates the configuration."""
  120. with self.m_lock:
  121. if self.m_cfg.enable and not new_cfg.enable:
  122. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  123. elif not self.m_cfg.enable and new_cfg.enable:
  124. self._log("Config dynamically updated: Group ENABLED.")
  125. else:
  126. self._log("Config dynamically updated: Parameters refreshed.")
  127. self.m_cfg = new_cfg
  128. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  129. if instance is None:
  130. return True
  131. cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
  132. if not cleaned:
  133. self._log(f"Cleanup retained for retry ({reason})")
  134. return cleaned
  135. def _cleanup_task(self, task: Task, reason: str = ""):
  136. if task and task.instance:
  137. with task._operation_lock:
  138. self._cleanup_instance(task.instance, reason, task.task_ref)
  139. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  140. removed = False
  141. with self.m_lock:
  142. if task in self.m_tasks:
  143. task.removed = True
  144. self.m_tasks.remove(task)
  145. removed = True
  146. if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
  147. self._leased_orders.discard(str(task.task_ref))
  148. self._on_task_removed(task)
  149. if cleanup and removed:
  150. self._cleanup_task(task, reason)
  151. return removed
  152. def _on_task_removed(self, task: Task):
  153. """Hook for subclasses when a task is removed (e.g. cleaning up caches)."""
  154. pass
  155. def _cleanup_all_tasks(self, reason: str = ""):
  156. with self.m_lock:
  157. tasks = list(self.m_tasks)
  158. self.m_tasks.clear()
  159. for task in tasks:
  160. task.removed = True
  161. if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
  162. self._leased_orders.discard(str(task.task_ref))
  163. self._on_all_tasks_cleaned()
  164. for task in tasks:
  165. self._cleanup_task(task, reason)
  166. def _on_all_tasks_cleaned(self):
  167. """Hook for subclasses when all tasks are cleared."""
  168. pass
  169. def _get_redis_key(self, routing_key: str) -> str:
  170. return f"vs:signal:{routing_key}"
  171. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  172. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  173. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  174. try:
  175. CloudApi.Instance().lock_account(acc.id, 0)
  176. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  177. except Exception as e:
  178. self._log(f"Failed to release account [{acc.username}]: {e}")
  179. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  180. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  181. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  182. return
  183. try:
  184. cloud = CloudApi.Instance()
  185. url = f"{cloud.base_url}/api/account/update"
  186. params = {"account_id": acc.id}
  187. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  188. payload = {
  189. "status": "disable",
  190. "next_use_time": far_future,
  191. "extra_data": {
  192. "disable_reason": reason,
  193. "disabled_at": int(time.time()),
  194. "module": self.TAG,
  195. "group": getattr(self.m_cfg, "identifier", "")
  196. }
  197. }
  198. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  199. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  200. except Exception as e:
  201. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  202. try:
  203. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  204. except Exception as lock_err:
  205. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  206. def _dispatch_error_policy(
  207. self,
  208. action: str,
  209. err: Exception,
  210. task: Optional[Task] = None,
  211. plg_cfg: Optional[PluginConfig] = None,
  212. task_id: Optional[int] = None
  213. ) -> None:
  214. """
  215. Booker error dispatch and self-healing policy engine:
  216. Executes self-healing actions based on the (action, error type) matrix:
  217. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account or fails order task, never releases back to pool.
  218. - 429 during spawn/login: Account-level rate limit. Locks account, updates tracker & cooldown.
  219. - 429 during book: Booking slot rate limit or egress IP throttle. Preserves account, removes task to rotate proxy.
  220. - 401 / session expired: Session invalidated. Removes task.
  221. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  222. - Handles logging and timer backoffs internally.
  223. """
  224. biz_err = normalize_error(err)
  225. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  226. acc = active_plg_cfg.account if active_plg_cfg else None
  227. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  228. if isinstance(biz_err, AccountStateError):
  229. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  230. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  231. # If bound order task, update VAS task status to failed
  232. if effective_task_id is not None:
  233. try:
  234. CloudApi.Instance().update_vas_task(str(effective_task_id), {
  235. "status": "failed",
  236. "fail_reason": f"Account fatal error: {biz_err.message} (code={biz_err.code})"
  237. })
  238. self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
  239. except Exception as update_err:
  240. self._log(f"Failed to update VAS task status: {update_err}")
  241. else:
  242. self.redis_client.zrem(self.m_tracker_key, effective_task_id)
  243. # If pool account, permanently disable it in cloud
  244. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  245. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  246. # Never release account back to pool
  247. if task:
  248. self._remove_task(task, f"account fatal error: {biz_err.message}")
  249. if action in ("spawn", "login", "create_session"):
  250. self.m_spawn_cooldown_until = time.time() + 5.0
  251. # 1. Target site rate limit / 429
  252. elif isinstance(biz_err, RateLimiteddError):
  253. if action in ("spawn", "login", "create_session"):
  254. backoff = float(self.m_cfg.login_backoff)
  255. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  256. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  257. try:
  258. CloudApi.Instance().lock_account(acc.id, int(backoff))
  259. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  260. except Exception as lock_err:
  261. self._log(f"Failed to lock account via cloud API: {lock_err}")
  262. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  263. if effective_task_id is not None:
  264. self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
  265. self.m_spawn_cooldown_until = time.time() + backoff
  266. if task:
  267. self._remove_task(task, "spawn account rate limited")
  268. elif action == "book":
  269. # 429 during booking: slot rate limit or IP restriction, preserve account and remove task to rotate proxy
  270. self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
  271. if task:
  272. self._remove_task(task, "booking rate limited")
  273. else:
  274. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  275. if task:
  276. self._remove_task(task, f"{action} rate limited")
  277. # 2. Session expired / 401
  278. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  279. if action in ("spawn", "login", "create_session"):
  280. self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
  281. self.m_spawn_cooldown_until = time.time() + 5.0
  282. else:
  283. self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
  284. if task:
  285. self._remove_task(task, f"{action} session expired")
  286. # 3. Permission denied / WAF blocked / 403
  287. elif isinstance(biz_err, PermissionDeniedError):
  288. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  289. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  290. if task:
  291. self._remove_task(task, f"{action} waf blocked")
  292. if action in ("spawn", "login", "create_session"):
  293. self._release_account_safely(acc, reason="WAF Blocked")
  294. self.m_spawn_cooldown_until = time.time() + proxy_cd
  295. # 4. Network or browser timeout
  296. elif isinstance(biz_err, NetworkTimeoutError):
  297. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  298. if task:
  299. task.next_run = time.time() + 5.0
  300. if action in ("spawn", "login", "create_session"):
  301. self._release_account_safely(acc, reason="Network Timeout")
  302. self.m_spawn_cooldown_until = time.time() + 10.0
  303. # 5. Fallback policy
  304. else:
  305. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  306. if action in ("spawn", "login", "create_session"):
  307. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  308. self.m_spawn_cooldown_until = time.time() + 10.0
  309. elif task:
  310. task.next_run = time.time() + 10.0
  311. # 6. Proxy health metrics recording
  312. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  313. ProxyManager.Instance().mark_proxy_result(
  314. proxy=active_plg_cfg.proxy.model_dump(),
  315. plugin_name=self.m_cfg.plugin_config.plugin_name,
  316. action=action,
  317. success=False,
  318. http_code=biz_err.http_status,
  319. error_type=biz_err.message
  320. )
  321. def _is_within_active_hours(self) -> bool:
  322. """Checks if current time is within allowed operating hours (UTC)."""
  323. start_str = self.m_cfg.active_time_start
  324. end_str = self.m_cfg.active_time_end
  325. if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
  326. return True
  327. current_utc_time = datetime.now(timezone.utc).time()
  328. start_time = datetime.strptime(start_str, "%H:%M").time()
  329. end_time = datetime.strptime(end_str, "%H:%M").time()
  330. if start_time <= end_time:
  331. return start_time <= current_utc_time <= end_time
  332. else:
  333. return current_utc_time >= start_time or current_utc_time <= end_time
  334. def _lease_order(self, task_id):
  335. with self.m_lock:
  336. self._leased_orders.add(str(task_id))
  337. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120})
  338. def _renew_leases(self):
  339. with self.m_lock:
  340. if self._leased_orders:
  341. self.redis_client.bulk_zadd(self.m_tracker_key, {
  342. tid: time.time() + 120 for tid in self._leased_orders})
  343. def _heartbeat_loop(self):
  344. # No browser calls here: long OTP/HTTP waits must not stop order leases.
  345. while not self._heartbeat_stop_event.wait(10):
  346. self._renew_leases()
  347. def _sync_successes(self):
  348. try:
  349. with self.m_lock:
  350. pending = dict(self._pending_results)
  351. for task_id, payload in pending.items():
  352. saved = False
  353. try:
  354. self._outbox.put(task_id, self.m_tracker_key, payload)
  355. saved = True
  356. except Exception as exc:
  357. self._log(f"Local success journal unavailable for task={task_id}: {exc}")
  358. saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved
  359. if saved:
  360. with self.m_lock:
  361. self._pending_results.pop(task_id, None)
  362. self._leased_orders.discard(task_id)
  363. self._outbox.flush(CloudApi.Instance(), self.redis_client._client,
  364. self._log, tracker=self.m_tracker_key)
  365. except Exception as exc:
  366. self._log(f"Success outbox retry failed: {exc}")
  367. def _success_sync_loop(self):
  368. while not self.m_stop_event.is_set() or self._pending_results:
  369. self._sync_successes()
  370. if self.m_stop_event.is_set() and not self._pending_results:
  371. break
  372. if self.m_stop_event.is_set():
  373. self._heartbeat_stop_event.wait(5)
  374. else:
  375. self.m_stop_event.wait(5)
  376. def _maintain_task(self, task):
  377. try:
  378. with task._operation_lock, self._resources.operation(task.instance):
  379. if task.removed or self.m_stop_event.is_set():
  380. return
  381. task.instance.keep_alive()
  382. if task.instance.health_check():
  383. task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5))
  384. else:
  385. self._remove_task(task, "unhealthy or keep-alive failed")
  386. except Exception as exc:
  387. task.next_remote_ping = time.time() + 5
  388. self._dispatch_error_policy("keep_alive", exc, task=task)
  389. finally:
  390. with self.m_lock:
  391. task.busy = False
  392. def _schedule_maintenance(self):
  393. with self.m_lock:
  394. if self.m_stop_event.is_set():
  395. return
  396. for task in self.m_tasks:
  397. if task.busy or task.removed or time.time() < task.next_remote_ping:
  398. continue
  399. task.busy = True
  400. if not self._executor.submit(self._maintain_task, task):
  401. task.busy = False
  402. def _maintain_loop(self):
  403. while not self.m_stop_event.wait(1):
  404. self._schedule_maintenance()
  405. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  406. """Checks if available dates in query_result match task preferences (defaults to True)."""
  407. return True
  408. def _run_book_job(self, task, query_result):
  409. try:
  410. with task._operation_lock, self._resources.operation(task.instance):
  411. if not task.removed and not self.m_stop_event.is_set():
  412. self._execute_book_job(task, query_result)
  413. finally:
  414. with self.m_lock:
  415. task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown)
  416. task.busy = False
  417. def _poll_booking_signals(self):
  418. now = time.time()
  419. with self.m_lock:
  420. ready = [t for t in self.m_tasks if not t.busy and not t.removed
  421. and t.book_allowed and now >= t.next_run]
  422. keys = {key for t in ready for key in t.acceptable_routing_keys}
  423. apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys]
  424. if not apt_types:
  425. return
  426. redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types]
  427. signals = self.redis_client.mget(redis_keys)
  428. for apt_type, raw_data in zip(apt_types, signals):
  429. if not raw_data:
  430. continue
  431. try:
  432. data = json.loads(raw_data)
  433. result = QueryResult.model_validate(data['query_result'])
  434. result.apt_type = AppointmentType.model_validate(data['apt_type'])
  435. if result.apt_type.routing_key != apt_type.routing_key:
  436. raise ValueError('signal routing key mismatch')
  437. except Exception as exc:
  438. # A producer may have replaced this value since MGET. Do not
  439. # delete a newer signal based on an old malformed payload.
  440. self._log(f"Invalid signal for {apt_type.routing_key}: {exc}")
  441. continue
  442. with self.m_lock:
  443. for task in ready:
  444. if (task.busy or task.removed or self.m_stop_event.is_set()
  445. or apt_type.routing_key not in task.acceptable_routing_keys
  446. or not self._is_date_of_interest(task, result)):
  447. continue
  448. task.busy = True
  449. if not self._executor.submit(self._run_book_job, task, result):
  450. task.busy = False
  451. def _booking_trigger_loop(self):
  452. self._log("Trigger loop started.")
  453. while not self.m_stop_event.is_set():
  454. try:
  455. if self.m_cfg.enable and self._is_within_active_hours():
  456. self._poll_booking_signals()
  457. except Exception as exc:
  458. self._log(f"Booking trigger loop exception: {exc}")
  459. self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval)
  460. def _execute_book_job(self, task: Task, query_result: QueryResult):
  461. raise NotImplementedError
  462. def _creator_loop(self):
  463. raise NotImplementedError
  464. def _push_success_notification(self, task_id, order_id, book_res):
  465. """Shared helper to sync booking success state to cloud and dispatch notifications."""
  466. grab_info = {
  467. "account": book_res.account,
  468. "session_id": book_res.session_id,
  469. "urn": book_res.urn,
  470. "slot_date": book_res.book_date,
  471. "slot_time": book_res.book_time,
  472. "timestamp": int(time.time()),
  473. "payment_link": book_res.payment_link
  474. }
  475. payload = {"status": "grabbed", "grabbed_history": grab_info}
  476. # Commit locally before retiring the browser or dropping its lease.
  477. with self.m_lock:
  478. self._pending_results[str(task_id)] = payload
  479. self._sync_successes()
  480. try:
  481. notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
  482. except Exception as exc:
  483. self._log(f"Booking succeeded, but notification could not be queued: {exc}")
  484. class BuiltinBookerGCO(BaseBookerGCO):
  485. """
  486. Unbound mode (shared built-in account pool):
  487. - Maintains target_instances count of instances globally.
  488. - Instances stay warm; when a slot signal is detected, orders are popped on-demand from the cloud queue.
  489. """
  490. TAG = "BUILTIN-BOOKER"
  491. def _creator_loop(self):
  492. self._log("Creator loop started.")
  493. while not self.m_stop_event.wait(1.0):
  494. if not self._is_within_active_hours():
  495. continue
  496. if time.time() < self.m_spawn_cooldown_until:
  497. continue
  498. with self.m_lock:
  499. current = len(self.m_tasks)
  500. target = self.m_cfg.booker.target_instances
  501. if current < target:
  502. self._spawn_worker()
  503. def _spawn_worker(self):
  504. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  505. return
  506. instance = None
  507. task = None
  508. success = False
  509. plg_cfg = None
  510. try:
  511. plg_cfg = PluginConfig()
  512. plg_cfg.debug = self.m_cfg.debug
  513. plg_cfg.free_config = self.m_cfg.free_config
  514. plg_cfg.session_max_life = self.m_cfg.session_max_life
  515. if self.m_cfg.need_account:
  516. acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  517. plg_cfg.account = type(plg_cfg.account)(**acc)
  518. if self.m_cfg.need_proxy:
  519. proxy = ProxyManager.Instance().get_next_proxy(
  520. self.m_cfg.proxy_pool,
  521. self.m_cfg.proxy_cd,
  522. plugin_name=self.m_cfg.plugin_config.plugin_name
  523. )
  524. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  525. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  526. self._resources.track(instance)
  527. with self._resources.operation(instance):
  528. instance.set_log(self.m_logger)
  529. instance.set_config(plg_cfg)
  530. instance.create_session()
  531. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  532. task = Task(
  533. instance=instance,
  534. next_run=time.time(),
  535. task_ref=None,
  536. acceptable_routing_keys=all_keys,
  537. source_queue="built-in",
  538. book_allowed=True,
  539. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  540. )
  541. with self.m_lock:
  542. if self.m_stop_event.is_set():
  543. return
  544. self.m_tasks.append(task)
  545. success = True
  546. if plg_cfg.proxy.port > 0:
  547. ProxyManager.Instance().mark_proxy_result(
  548. proxy=plg_cfg.proxy.model_dump(),
  549. plugin_name=self.m_cfg.plugin_config.plugin_name,
  550. action="spawn",
  551. success=True,
  552. http_code=200
  553. )
  554. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  555. except Exception as e:
  556. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  557. finally:
  558. if not success and instance:
  559. self._cleanup_instance(instance, "spawn failed")
  560. def _execute_book_job(self, task: Task, query_result: QueryResult):
  561. queue_name = f"auto.{query_result.apt_type.routing_key}"
  562. task_data = None
  563. try:
  564. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  565. except BizLogicError:
  566. # Queue has no pending tasks, return normally
  567. return
  568. except Exception as e:
  569. self._log(f"⚠️ [CloudApi] Failed to pop task from {queue_name}: {e}")
  570. return
  571. if not task_data:
  572. return
  573. task_id = task_data.get('id')
  574. order_id = task_data.get('order_id')
  575. self._lease_order(task_id)
  576. book_res = None
  577. try:
  578. user_input = task_data.get('user_inputs', {})
  579. book_res = task.instance.book(query_result, user_input)
  580. if book_res.success:
  581. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  582. self._push_success_notification(task_id, order_id, book_res)
  583. task.successful_bookings += 1
  584. max_b = self.m_cfg.booker.max_bookings_per_account
  585. if max_b > 0 and task.successful_bookings >= max_b:
  586. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  587. self._remove_task(task, "max bookings reached")
  588. else:
  589. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  590. except Exception as e:
  591. self._dispatch_error_policy("book", e, task=task, task_id=task_id)
  592. finally:
  593. with self.m_lock:
  594. if str(task_id) not in self._pending_results:
  595. self._leased_orders.discard(str(task_id))
  596. if book_res is None or not book_res.success:
  597. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()})
  598. class OrderBookerGCO(BaseBookerGCO):
  599. """
  600. Bound mode (order provides its own account credentials):
  601. - Maintains warm-up quotas per routing queue.
  602. - Strict 1:1 binding: one instance per cloud order.
  603. - Instance is destroyed immediately after booking completes.
  604. """
  605. TAG = "ORDER-BOOKER"
  606. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  607. super().__init__(cfg, redis_conf, logger)
  608. self.m_task_data_cache: Dict[str, dict] = {}
  609. def _start_additional_threads(self):
  610. thread = threading.Thread(target=self._cache_refresh_loop, daemon=True)
  611. self._background_threads.append(thread)
  612. thread.start()
  613. def _on_task_removed(self, task: Task):
  614. task_id = task.task_ref
  615. if task_id is not None:
  616. self.m_task_data_cache.pop(str(task_id), None)
  617. def _on_all_tasks_cleaned(self):
  618. self.m_task_data_cache.clear()
  619. def _cache_refresh_loop(self):
  620. self._log("Cache refresh loop started.")
  621. refresh_interval = 15 * 60
  622. while not self.m_stop_event.is_set():
  623. try:
  624. if self.m_stop_event.wait(1):
  625. break
  626. with self.m_lock:
  627. tasks_to_check = {
  628. tid: data.get('_last_refresh', 0)
  629. for tid, data in self.m_task_data_cache.items()
  630. }
  631. if not tasks_to_check:
  632. continue
  633. now = time.time()
  634. for tid, last_refresh in tasks_to_check.items():
  635. if self.m_stop_event.is_set():
  636. break
  637. if now - last_refresh >= refresh_interval:
  638. fresh_data = CloudApi.Instance().get_vas_task(tid)
  639. if fresh_data:
  640. fresh_data['_last_refresh'] = time.time()
  641. with self.m_lock:
  642. if tid in self.m_task_data_cache:
  643. self.m_task_data_cache[tid] = fresh_data
  644. self.m_stop_event.wait(0.5)
  645. except Exception as e:
  646. self._log(f'Cache refresh loop exception: {e}')
  647. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  648. if query_result.availability_status != AvailabilityStatus.Available:
  649. return True
  650. task_id = task.task_ref
  651. task_data = self.m_task_data_cache.get(str(task_id), {})
  652. user_input = task_data.get('user_inputs', {})
  653. expected_end_date = (
  654. user_input.get('expected_end_date')
  655. or '2100-01-01'
  656. )
  657. available_date = query_result.earliest_date
  658. if available_date is None:
  659. return False
  660. dt = available_date.strftime("%Y-%m-%d")
  661. return dt <= expected_end_date
  662. def _execute_book_job(self, task: Task, query_result: QueryResult):
  663. task_id = task.task_ref
  664. task_data = None
  665. try:
  666. with self.m_lock:
  667. task_data = self.m_task_data_cache.get(str(task_id))
  668. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']:
  669. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  670. self._remove_task(task, "bound task no longer valid")
  671. self.redis_client.zrem(self.m_tracker_key, task_id)
  672. return
  673. order_id = task_data.get('order_id')
  674. user_input = task_data.get('user_inputs', {})
  675. book_res = task.instance.book(query_result, user_input)
  676. if book_res.success:
  677. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  678. self._push_success_notification(task_id, order_id, book_res)
  679. self._remove_task(task, "booking success")
  680. else:
  681. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  682. except Exception as e:
  683. self._dispatch_error_policy("book", e, task=task)
  684. def _creator_loop(self):
  685. self._log("Creator loop started.")
  686. while not self.m_stop_event.wait(1.0):
  687. if not self._is_within_active_hours():
  688. continue
  689. if time.time() < self.m_spawn_cooldown_until:
  690. continue
  691. for apt in self.m_cfg.appointment_types:
  692. r_key = apt.routing_key
  693. with self.m_lock:
  694. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  695. target = self.m_cfg.booker.target_instances
  696. if active < target:
  697. self._spawn_worker(r_key)
  698. def _spawn_worker(self, target_routing_key: str):
  699. if self.m_stop_event.is_set() or not self._resources.retry_failed():
  700. return
  701. instance = None
  702. task = None
  703. success = False
  704. task_id = None
  705. plg_cfg = None
  706. try:
  707. queue_name = f"auto.{target_routing_key}"
  708. try:
  709. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  710. except BizLogicError:
  711. # Queue has no pending tasks, return directly
  712. return
  713. if not task_data:
  714. return
  715. task_id = task_data['id']
  716. with self.m_lock:
  717. self.m_task_data_cache[str(task_id)] = task_data
  718. self._lease_order(task_id)
  719. user_inputs = task_data.get('user_inputs', {})
  720. plg_cfg = PluginConfig()
  721. plg_cfg.debug = self.m_cfg.debug
  722. plg_cfg.free_config = self.m_cfg.free_config
  723. plg_cfg.session_max_life = self.m_cfg.session_max_life
  724. plg_cfg.account.username = user_inputs.get("username", "")
  725. plg_cfg.account.password = user_inputs.get("password", "")
  726. if not plg_cfg.account.username:
  727. raise AccountIncompleteError(f"Task={task_id} missing username credentials in user_inputs")
  728. acceptable_keys = [target_routing_key]
  729. if self.m_cfg.need_proxy:
  730. proxy = ProxyManager.Instance().get_next_proxy(
  731. self.m_cfg.proxy_pool,
  732. self.m_cfg.proxy_cd,
  733. plugin_name=self.m_cfg.plugin_config.plugin_name
  734. )
  735. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  736. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  737. self._resources.track(instance)
  738. with self._resources.operation(instance):
  739. instance.set_log(self.m_logger)
  740. instance.set_config(plg_cfg)
  741. instance.create_session()
  742. task = Task(
  743. instance=instance,
  744. next_run=time.time(),
  745. task_ref=task_id,
  746. acceptable_routing_keys=acceptable_keys,
  747. source_queue=target_routing_key,
  748. book_allowed=True,
  749. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  750. )
  751. with self.m_lock:
  752. if self.m_stop_event.is_set():
  753. return
  754. self.m_tasks.append(task)
  755. success = True
  756. if plg_cfg.proxy.port > 0:
  757. ProxyManager.Instance().mark_proxy_result(
  758. proxy=plg_cfg.proxy.model_dump(),
  759. plugin_name=self.m_cfg.plugin_config.plugin_name,
  760. action="spawn",
  761. success=True,
  762. http_code=200
  763. )
  764. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  765. except Exception as e:
  766. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
  767. finally:
  768. if not success and instance:
  769. self._cleanup_instance(instance, "spawn failed")
  770. if not success and task_id is not None:
  771. with self.m_lock:
  772. self.m_task_data_cache.pop(str(task_id), None)
  773. try:
  774. with self.m_lock:
  775. self._leased_orders.discard(str(task_id))
  776. # Retain a recovery deadline (including any 429 cooldown).
  777. self.redis_client.zadd(self.m_tracker_key, {
  778. str(task_id): max(time.time(), self.m_spawn_cooldown_until)})
  779. except Exception:
  780. pass