booker.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775
  1. import os
  2. import time
  3. import json
  4. import threading
  5. import random
  6. from datetime import datetime, timedelta, timezone
  7. from typing import List, Dict, Callable, Optional
  8. from core_types import (
  9. GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult,
  10. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
  11. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  12. NetworkTimeoutError, BizLogicError, AccountStateError, AccountAuthError,
  13. AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error
  14. )
  15. from plugin_factory import PluginFactory
  16. from toolkit.cloud_api import CloudApi
  17. from toolkit.proxy_manager import ProxyManager
  18. from toolkit.thread_pool import ThreadPool
  19. from toolkit.notifier import notify_book_result
  20. from utils.safe_redis_cli import SafeRedisClient
  21. class BaseBookerGCO:
  22. """
  23. Base Booker class encapsulating shared infrastructure, lifecycle management, and logic loops.
  24. """
  25. TAG = "BOOKER"
  26. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  27. self.m_cfg = cfg
  28. self.m_factory = PluginFactory()
  29. self.m_logger = logger
  30. self.m_tasks: List[Task] = []
  31. self.m_lock = threading.RLock()
  32. self.m_stop_event = threading.Event()
  33. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  34. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  35. self.m_last_group_query_time = 0.0
  36. self.m_spawn_cooldown_until = 0.0
  37. def _log(self, message):
  38. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  39. if self.m_logger:
  40. self.m_logger(f'{prefix} {message}')
  41. else:
  42. print(f'{prefix} {message}')
  43. def get_status(self) -> GroupStatus:
  44. with self.m_lock:
  45. tasks = list(self.m_tasks)
  46. gid = self.m_cfg.identifier
  47. enable = self.m_cfg.enable
  48. plugin_name = self.m_cfg.plugin_config.plugin_name
  49. plugins_status = []
  50. healthy_count = 0
  51. for t in tasks:
  52. if t and t.instance:
  53. st = t.instance.get_plugin_status()
  54. if st.is_healthy:
  55. healthy_count += 1
  56. plugins_status.append(st)
  57. return GroupStatus(
  58. group_id=gid,
  59. enable=enable,
  60. plugin_name=plugin_name,
  61. total_tasks=len(tasks),
  62. healthy_tasks=healthy_count,
  63. plugins=plugins_status,
  64. updated_at=utc_now_str(),
  65. )
  66. def start(self):
  67. if not self.m_cfg.enable:
  68. return
  69. self._log(f"Starting {self.TAG}...")
  70. plugin_name = self.m_cfg.plugin_config.plugin_name
  71. class_name = "".join(part.title() for part in plugin_name.split('_'))
  72. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  73. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  74. threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
  75. threading.Thread(target=self._creator_loop, daemon=True).start()
  76. threading.Thread(target=self._maintain_loop, daemon=True).start()
  77. self._start_additional_threads()
  78. def _start_additional_threads(self):
  79. """Hook for subclasses to start additional background threads."""
  80. pass
  81. def stop(self):
  82. self._log("Stopping Booker...")
  83. self.m_stop_event.set()
  84. self._cleanup_all_tasks("booker stop")
  85. def update_config(self, new_cfg: GroupConfig):
  86. """Dynamically updates the configuration."""
  87. with self.m_lock:
  88. if self.m_cfg.enable and not new_cfg.enable:
  89. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  90. elif not self.m_cfg.enable and new_cfg.enable:
  91. self._log("Config dynamically updated: Group ENABLED.")
  92. else:
  93. self._log("Config dynamically updated: Parameters refreshed.")
  94. self.m_cfg = new_cfg
  95. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  96. try:
  97. if instance:
  98. ref_str = f" for task={task_ref}" if task_ref else ""
  99. self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
  100. instance.cleanup()
  101. except Exception as e:
  102. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  103. def _cleanup_task(self, task: Task, reason: str = ""):
  104. if task and task.instance:
  105. self._cleanup_instance(task.instance, reason, task.task_ref)
  106. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  107. removed = False
  108. with self.m_lock:
  109. if task in self.m_tasks:
  110. self.m_tasks.remove(task)
  111. removed = True
  112. self._on_task_removed(task)
  113. if cleanup and removed:
  114. self._cleanup_task(task, reason)
  115. return removed
  116. def _on_task_removed(self, task: Task):
  117. """Hook for subclasses when a task is removed (e.g. cleaning up caches)."""
  118. pass
  119. def _cleanup_all_tasks(self, reason: str = ""):
  120. with self.m_lock:
  121. tasks = list(self.m_tasks)
  122. self.m_tasks.clear()
  123. self._on_all_tasks_cleaned()
  124. for task in tasks:
  125. self._cleanup_task(task, reason)
  126. def _on_all_tasks_cleaned(self):
  127. """Hook for subclasses when all tasks are cleared."""
  128. pass
  129. def _get_redis_key(self, routing_key: str) -> str:
  130. return f"vs:signal:{routing_key}"
  131. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  132. """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
  133. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  134. try:
  135. CloudApi.Instance().lock_account(acc.id, 0)
  136. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  137. except Exception as e:
  138. self._log(f"Failed to release account [{acc.username}]: {e}")
  139. def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
  140. """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
  141. if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
  142. return
  143. try:
  144. cloud = CloudApi.Instance()
  145. url = f"{cloud.base_url}/api/account/update"
  146. params = {"account_id": acc.id}
  147. far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
  148. payload = {
  149. "status": "disable",
  150. "next_use_time": far_future,
  151. "extra_data": {
  152. "disable_reason": reason,
  153. "disabled_at": int(time.time()),
  154. "module": self.TAG,
  155. "group": getattr(self.m_cfg, "identifier", "")
  156. }
  157. }
  158. cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
  159. self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
  160. except Exception as e:
  161. self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
  162. try:
  163. CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
  164. except Exception as lock_err:
  165. self._log(f"Fallback 10-year lock also failed: {lock_err}")
  166. def _dispatch_error_policy(
  167. self,
  168. action: str,
  169. err: Exception,
  170. task: Optional[Task] = None,
  171. plg_cfg: Optional[PluginConfig] = None,
  172. task_id: Optional[int] = None
  173. ) -> None:
  174. """
  175. Booker error dispatch and self-healing policy engine:
  176. Executes self-healing actions based on the (action, error type) matrix:
  177. - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account or fails order task, never releases back to pool.
  178. - 429 during spawn/login: Account-level rate limit. Locks account, updates tracker & cooldown.
  179. - 429 during book: Booking slot rate limit or egress IP throttle. Preserves account, removes task to rotate proxy.
  180. - 401 / session expired: Session invalidated. Removes task.
  181. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
  182. - Handles logging and timer backoffs internally.
  183. """
  184. biz_err = normalize_error(err)
  185. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  186. acc = active_plg_cfg.account if active_plg_cfg else None
  187. # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
  188. if isinstance(biz_err, AccountStateError):
  189. self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
  190. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  191. # If bound order task, update VAS task status to failed
  192. if effective_task_id is not None:
  193. try:
  194. CloudApi.Instance().update_vas_task(str(effective_task_id), {
  195. "status": "failed",
  196. "fail_reason": f"Account fatal error: {biz_err.message} (code={biz_err.code})"
  197. })
  198. self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
  199. except Exception as update_err:
  200. self._log(f"Failed to update VAS task status: {update_err}")
  201. try:
  202. self.redis_client.zrem(self.m_tracker_key, effective_task_id)
  203. except Exception as zrem_err:
  204. self._log(f"Failed to remove task from tracker: {zrem_err}")
  205. # If pool account, permanently disable it in cloud
  206. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  207. self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
  208. # Never release account back to pool
  209. if task:
  210. self._remove_task(task, f"account fatal error: {biz_err.message}")
  211. if action in ("spawn", "login", "create_session"):
  212. self.m_spawn_cooldown_until = time.time() + 5.0
  213. # 1. Target site rate limit / 429
  214. elif isinstance(biz_err, RateLimiteddError):
  215. if action in ("spawn", "login", "create_session"):
  216. backoff = float(self.m_cfg.login_backoff)
  217. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  218. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  219. try:
  220. CloudApi.Instance().lock_account(acc.id, int(backoff))
  221. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  222. except Exception as lock_err:
  223. self._log(f"Failed to lock account via cloud API: {lock_err}")
  224. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  225. if effective_task_id is not None:
  226. self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
  227. self.m_spawn_cooldown_until = time.time() + backoff
  228. if task:
  229. self._remove_task(task, "spawn account rate limited")
  230. elif action == "book":
  231. # 429 during booking: slot rate limit or IP restriction, preserve account and remove task to rotate proxy
  232. self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
  233. if task:
  234. self._remove_task(task, "booking rate limited")
  235. else:
  236. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  237. if task:
  238. self._remove_task(task, f"{action} rate limited")
  239. # 2. Session expired / 401
  240. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  241. if action in ("spawn", "login", "create_session"):
  242. self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
  243. self.m_spawn_cooldown_until = time.time() + 5.0
  244. else:
  245. self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
  246. if task:
  247. self._remove_task(task, f"{action} session expired")
  248. # 3. Permission denied / WAF blocked / 403
  249. elif isinstance(biz_err, PermissionDeniedError):
  250. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  251. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  252. if task:
  253. self._remove_task(task, f"{action} waf blocked")
  254. if action in ("spawn", "login", "create_session"):
  255. self._release_account_safely(acc, reason="WAF Blocked")
  256. self.m_spawn_cooldown_until = time.time() + proxy_cd
  257. # 4. Network or browser timeout
  258. elif isinstance(biz_err, NetworkTimeoutError):
  259. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  260. if task:
  261. task.next_run = time.time() + 5.0
  262. if action in ("spawn", "login", "create_session"):
  263. self._release_account_safely(acc, reason="Network Timeout")
  264. self.m_spawn_cooldown_until = time.time() + 10.0
  265. # 5. Fallback policy
  266. else:
  267. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  268. if action in ("spawn", "login", "create_session"):
  269. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  270. self.m_spawn_cooldown_until = time.time() + 10.0
  271. elif task:
  272. task.next_run = time.time() + 10.0
  273. # 6. Proxy health metrics recording
  274. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  275. ProxyManager.Instance().mark_proxy_result(
  276. proxy=active_plg_cfg.proxy.model_dump(),
  277. plugin_name=self.m_cfg.plugin_config.plugin_name,
  278. action=action,
  279. success=False,
  280. http_code=biz_err.http_status,
  281. error_type=biz_err.message
  282. )
  283. def _is_within_active_hours(self) -> bool:
  284. """Checks if current time is within allowed operating hours."""
  285. start_str = self.m_cfg.active_time_start
  286. end_str = self.m_cfg.active_time_end
  287. current_bj_time = datetime.utcnow().time()
  288. start_time = datetime.strptime(start_str, "%H:%M").time()
  289. end_time = datetime.strptime(end_str, "%H:%M").time()
  290. return start_time <= current_bj_time <= end_time
  291. def _maintain_loop(self):
  292. self._log("Maintain loop started.")
  293. while not self.m_stop_event.is_set():
  294. try:
  295. time.sleep(1.0)
  296. with self.m_lock:
  297. tasks_to_check = list(self.m_tasks)
  298. if not tasks_to_check:
  299. continue
  300. dead_tasks = []
  301. healthy_tasks = []
  302. now = time.time()
  303. for t in tasks_to_check:
  304. if now >= t.next_remote_ping:
  305. t.instance.keep_alive()
  306. if t.instance.health_check():
  307. healthy_tasks.append(t)
  308. t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
  309. else:
  310. dead_tasks.append(t)
  311. else:
  312. healthy_tasks.append(t)
  313. self._on_maintain_ping(healthy_tasks, dead_tasks)
  314. if dead_tasks:
  315. for t in dead_tasks:
  316. self._remove_task(t, "unhealthy or keep-alive failed")
  317. except Exception as e:
  318. self._log(f'Maintain loop exception: {e}')
  319. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  320. """Subclass hook for additional handling of healthy/dead tasks during maintenance."""
  321. pass
  322. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  323. """Checks if available dates in query_result match task preferences (defaults to True)."""
  324. return True
  325. def _booking_trigger_loop(self):
  326. self._log("Trigger loop started.")
  327. while not self.m_stop_event.is_set():
  328. try:
  329. time.sleep(0.1)
  330. now = time.time()
  331. for apt_type in self.m_cfg.appointment_types:
  332. redis_key = self._get_redis_key(apt_type.routing_key)
  333. raw_data = self.redis_client.get(redis_key)
  334. if not raw_data:
  335. continue
  336. try:
  337. data = json.loads(raw_data)
  338. query_result = QueryResult.model_validate(data['query_result'])
  339. query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
  340. except Exception as parse_err:
  341. self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
  342. self.redis_client.delete(redis_key)
  343. continue
  344. matching_tasks = []
  345. with self.m_lock:
  346. for task in self.m_tasks:
  347. if now < task.next_run or not task.book_allowed:
  348. continue
  349. if apt_type.routing_key not in task.acceptable_routing_keys:
  350. continue
  351. if not self._is_date_of_interest(task, query_result):
  352. continue
  353. task.next_run = now + self.m_cfg.booker.booking_cooldown
  354. matching_tasks.append(task)
  355. if matching_tasks:
  356. threads = []
  357. for task in matching_tasks:
  358. self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
  359. t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
  360. threads.append(t)
  361. t.start()
  362. for t in threads:
  363. t.join()
  364. except Exception as e:
  365. self._log(f"Booking trigger loop exception: {e}")
  366. def _execute_book_job(self, task: Task, query_result: QueryResult):
  367. raise NotImplementedError
  368. def _creator_loop(self):
  369. raise NotImplementedError
  370. def _push_success_notification(self, task_id, order_id, book_res):
  371. """Shared helper to sync booking success state to cloud and dispatch notifications."""
  372. grab_info = {
  373. "account": book_res.account,
  374. "session_id": book_res.session_id,
  375. "urn": book_res.urn,
  376. "slot_date": book_res.book_date,
  377. "slot_time": book_res.book_time,
  378. "timestamp": int(time.time()),
  379. "payment_link": book_res.payment_link
  380. }
  381. def _update_cloud_success():
  382. try:
  383. CloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
  384. except Exception as e:
  385. self._log(f"Failed to update success state to cloud: {e}")
  386. ThreadPool.getInstance().enqueue(_update_cloud_success)
  387. notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
  388. self.redis_client.zrem(self.m_tracker_key, task_id)
  389. class BuiltinBookerGCO(BaseBookerGCO):
  390. """
  391. Unbound mode (shared built-in account pool):
  392. - Maintains target_instances count of instances globally.
  393. - Instances stay warm; when a slot signal is detected, orders are popped on-demand from the cloud queue.
  394. """
  395. TAG = "BUILTIN-BOOKER"
  396. def _creator_loop(self):
  397. self._log("Creator loop started.")
  398. while not self.m_stop_event.wait(1.0):
  399. if not self._is_within_active_hours():
  400. continue
  401. if time.time() < self.m_spawn_cooldown_until:
  402. continue
  403. with self.m_lock:
  404. current = len(self.m_tasks)
  405. target = self.m_cfg.booker.target_instances
  406. if current < target:
  407. self._spawn_worker()
  408. def _spawn_worker(self):
  409. instance = None
  410. task = None
  411. success = False
  412. plg_cfg = None
  413. try:
  414. plg_cfg = PluginConfig()
  415. plg_cfg.debug = self.m_cfg.debug
  416. plg_cfg.free_config = self.m_cfg.free_config
  417. plg_cfg.session_max_life = self.m_cfg.session_max_life
  418. if self.m_cfg.need_account:
  419. acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  420. plg_cfg.account = type(plg_cfg.account)(**acc)
  421. if self.m_cfg.need_proxy:
  422. proxy = ProxyManager.Instance().get_next_proxy(
  423. self.m_cfg.proxy_pool,
  424. self.m_cfg.proxy_cd,
  425. plugin_name=self.m_cfg.plugin_config.plugin_name
  426. )
  427. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  428. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  429. instance.set_log(self.m_logger)
  430. instance.set_config(plg_cfg)
  431. instance.create_session()
  432. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  433. task = Task(
  434. instance=instance,
  435. next_run=time.time(),
  436. task_ref=None,
  437. acceptable_routing_keys=all_keys,
  438. source_queue="built-in",
  439. book_allowed=True,
  440. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  441. )
  442. with self.m_lock:
  443. self.m_tasks.append(task)
  444. success = True
  445. if plg_cfg.proxy.port > 0:
  446. ProxyManager.Instance().mark_proxy_result(
  447. proxy=plg_cfg.proxy.model_dump(),
  448. plugin_name=self.m_cfg.plugin_config.plugin_name,
  449. action="spawn",
  450. success=True,
  451. http_code=200
  452. )
  453. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  454. except Exception as e:
  455. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  456. finally:
  457. if not success and instance:
  458. self._cleanup_instance(instance, "spawn failed")
  459. def _execute_book_job(self, task: Task, query_result: QueryResult):
  460. queue_name = f"auto.{query_result.apt_type.routing_key}"
  461. task_data = None
  462. try:
  463. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  464. except BizLogicError:
  465. # Queue has no pending tasks, return normally
  466. return
  467. except Exception as e:
  468. self._log(f"⚠️ [CloudApi] Failed to pop task from {queue_name}: {e}")
  469. return
  470. if not task_data:
  471. return
  472. task_id = task_data.get('id')
  473. order_id = task_data.get('order_id')
  474. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
  475. try:
  476. user_input = task_data.get('user_inputs', {})
  477. book_res = task.instance.book(query_result, user_input)
  478. if book_res.success:
  479. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  480. self._push_success_notification(task_id, order_id, book_res)
  481. task.successful_bookings += 1
  482. max_b = self.m_cfg.booker.max_bookings_per_account
  483. if max_b > 0 and task.successful_bookings >= max_b:
  484. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  485. self._remove_task(task, "max bookings reached")
  486. else:
  487. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  488. except Exception as e:
  489. self._dispatch_error_policy("book", e, task=task)
  490. class OrderBookerGCO(BaseBookerGCO):
  491. """
  492. Bound mode (order provides its own account credentials):
  493. - Maintains warm-up quotas per routing queue.
  494. - Strict 1:1 binding: one instance per cloud order.
  495. - Instance is destroyed immediately after booking completes.
  496. """
  497. TAG = "ORDER-BOOKER"
  498. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  499. super().__init__(cfg, redis_conf, logger)
  500. self.m_task_data_cache: Dict[str, dict] = {}
  501. self.heartbeat_ttl = 2 * 60.0
  502. def _start_additional_threads(self):
  503. threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
  504. def _on_task_removed(self, task: Task):
  505. task_id = task.task_ref
  506. if task_id:
  507. self.m_task_data_cache.pop(str(task_id), None)
  508. def _on_all_tasks_cleaned(self):
  509. self.m_task_data_cache.clear()
  510. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  511. if healthy_tasks:
  512. new_deadline = time.time() + self.heartbeat_ttl
  513. mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
  514. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  515. if dead_tasks:
  516. mapping = {str(t.task_ref): 0 for t in dead_tasks}
  517. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  518. def _cache_refresh_loop(self):
  519. self._log("Cache refresh loop started.")
  520. refresh_interval = 15 * 60
  521. while not self.m_stop_event.is_set():
  522. try:
  523. time.sleep(1)
  524. with self.m_lock:
  525. tasks_to_check = {
  526. tid: data.get('_last_refresh', 0)
  527. for tid, data in self.m_task_data_cache.items()
  528. }
  529. if not tasks_to_check:
  530. continue
  531. now = time.time()
  532. for tid, last_refresh in tasks_to_check.items():
  533. if now - last_refresh >= refresh_interval:
  534. fresh_data = CloudApi.Instance().get_vas_task(tid)
  535. if fresh_data:
  536. fresh_data['_last_refresh'] = time.time()
  537. with self.m_lock:
  538. if tid in self.m_task_data_cache:
  539. self.m_task_data_cache[tid] = fresh_data
  540. time.sleep(0.5)
  541. except Exception as e:
  542. self._log(f'Cache refresh loop exception: {e}')
  543. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  544. if query_result.availability_status != AvailabilityStatus.Available:
  545. return True
  546. task_id = task.task_ref
  547. task_data = self.m_task_data_cache.get(str(task_id), {})
  548. user_input = task_data.get('user_inputs', {})
  549. expected_end_date = (
  550. user_input.get('expected_end_date')
  551. or '2100-01-01'
  552. )
  553. available_date = query_result.earliest_date
  554. dt = available_date.strftime("%Y-%m-%d")
  555. return dt <= expected_end_date
  556. def _execute_book_job(self, task: Task, query_result: QueryResult):
  557. task_id = task.task_ref
  558. task_data = None
  559. try:
  560. with self.m_lock:
  561. task_data = self.m_task_data_cache.get(str(task_id))
  562. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
  563. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  564. self._remove_task(task, "bound task no longer valid")
  565. self.redis_client.zrem(self.m_tracker_key, task_id)
  566. return
  567. order_id = task_data.get('order_id')
  568. user_input = task_data.get('user_inputs', {})
  569. book_res = task.instance.book(query_result, user_input)
  570. if book_res.success:
  571. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  572. self._push_success_notification(task_id, order_id, book_res)
  573. self._remove_task(task, "booking success")
  574. else:
  575. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  576. except Exception as e:
  577. self._dispatch_error_policy("book", e, task=task)
  578. def _creator_loop(self):
  579. self._log("Creator loop started.")
  580. while not self.m_stop_event.wait(1.0):
  581. if not self._is_within_active_hours():
  582. continue
  583. if time.time() < self.m_spawn_cooldown_until:
  584. continue
  585. for apt in self.m_cfg.appointment_types:
  586. r_key = apt.routing_key
  587. with self.m_lock:
  588. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  589. target = self.m_cfg.booker.target_instances
  590. if active < target:
  591. self._spawn_worker(r_key)
  592. def _spawn_worker(self, target_routing_key: str):
  593. instance = None
  594. task = None
  595. success = False
  596. task_id = None
  597. plg_cfg = None
  598. try:
  599. queue_name = f"auto.{target_routing_key}"
  600. try:
  601. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  602. except BizLogicError:
  603. # Queue has no pending tasks, return directly
  604. return
  605. if not task_data:
  606. return
  607. task_id = task_data['id']
  608. with self.m_lock:
  609. self.m_task_data_cache[str(task_id)] = task_data
  610. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
  611. user_inputs = task_data.get('user_inputs', {})
  612. plg_cfg = PluginConfig()
  613. plg_cfg.debug = self.m_cfg.debug
  614. plg_cfg.free_config = self.m_cfg.free_config
  615. plg_cfg.session_max_life = self.m_cfg.session_max_life
  616. plg_cfg.account.username = user_inputs.get("username", "")
  617. plg_cfg.account.password = user_inputs.get("password", "")
  618. if not plg_cfg.account.username:
  619. raise AccountIncompleteError(f"Task={task_id} missing username credentials in user_inputs")
  620. acceptable_keys = [target_routing_key]
  621. if self.m_cfg.need_proxy:
  622. proxy = ProxyManager.Instance().get_next_proxy(
  623. self.m_cfg.proxy_pool,
  624. self.m_cfg.proxy_cd,
  625. plugin_name=self.m_cfg.plugin_config.plugin_name
  626. )
  627. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  628. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  629. instance.set_log(self.m_logger)
  630. instance.set_config(plg_cfg)
  631. instance.create_session()
  632. task = Task(
  633. instance=instance,
  634. next_run=time.time(),
  635. task_ref=task_id,
  636. acceptable_routing_keys=acceptable_keys,
  637. source_queue=target_routing_key,
  638. book_allowed=True,
  639. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  640. )
  641. with self.m_lock:
  642. self.m_tasks.append(task)
  643. success = True
  644. if plg_cfg.proxy.port > 0:
  645. ProxyManager.Instance().mark_proxy_result(
  646. proxy=plg_cfg.proxy.model_dump(),
  647. plugin_name=self.m_cfg.plugin_config.plugin_name,
  648. action="spawn",
  649. success=True,
  650. http_code=200
  651. )
  652. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  653. except Exception as e:
  654. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
  655. finally:
  656. if not success and instance:
  657. self._cleanup_instance(instance, "spawn failed")
  658. if not success and task_id:
  659. with self.m_lock:
  660. self.m_task_data_cache.pop(str(task_id), None)
  661. try:
  662. self.redis_client.zrem(self.m_tracker_key, task_id)
  663. except Exception:
  664. pass