booker.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672
  1. import os
  2. import time
  3. import json
  4. import threading
  5. import random
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from core_types import (
  9. GroupConfig, PluginConfig, Task, BookResult, QueryResult,
  10. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
  11. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  12. NetworkTimeoutError
  13. )
  14. from plugin_factory import PluginFactory
  15. from toolkit.cloud_api import CloudApi
  16. from toolkit.proxy_manager import ProxyManager
  17. from toolkit.thread_pool import ThreadPool
  18. from toolkit.notifier import notify_book_result
  19. from utils.safe_redis_cli import SafeRedisClient
  20. class BaseBookerGCO:
  21. """
  22. Booker 基类,封装公共的基础设施与通用的生命周期管理、逻辑循环等。
  23. """
  24. TAG = "BOOKER"
  25. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  26. self.m_cfg = cfg
  27. self.m_factory = PluginFactory()
  28. self.m_logger = logger
  29. self.m_tasks: List[Task] = []
  30. self.m_lock = threading.RLock()
  31. self.m_stop_event = threading.Event()
  32. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  33. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  34. self.m_last_group_query_time = 0.0
  35. self.m_spawn_cooldown_until = 0.0
  36. def _log(self, message):
  37. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  38. if self.m_logger:
  39. self.m_logger(f'{prefix} {message}')
  40. else:
  41. print(f'{prefix} {message}')
  42. def get_status(self) -> GroupStatus:
  43. with self.m_lock:
  44. tasks = list(self.m_tasks)
  45. gid = self.m_cfg.identifier
  46. enable = self.m_cfg.enable
  47. plugin_name = self.m_cfg.plugin_config.plugin_name
  48. plugins_status = []
  49. healthy_count = 0
  50. for t in tasks:
  51. if t and t.instance:
  52. st = t.instance.get_plugin_status()
  53. if st.is_healthy:
  54. healthy_count += 1
  55. plugins_status.append(st)
  56. return GroupStatus(
  57. group_id=gid,
  58. enable=enable,
  59. plugin_name=plugin_name,
  60. total_tasks=len(tasks),
  61. healthy_tasks=healthy_count,
  62. plugins=plugins_status,
  63. updated_at=utc_now_str(),
  64. )
  65. def start(self):
  66. if not self.m_cfg.enable:
  67. return
  68. self._log(f"Starting {self.TAG}...")
  69. plugin_name = self.m_cfg.plugin_config.plugin_name
  70. class_name = "".join(part.title() for part in plugin_name.split('_'))
  71. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  72. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  73. threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
  74. threading.Thread(target=self._creator_loop, daemon=True).start()
  75. threading.Thread(target=self._maintain_loop, daemon=True).start()
  76. self._start_additional_threads()
  77. def _start_additional_threads(self):
  78. """子类扩展线程的 Hook"""
  79. pass
  80. def stop(self):
  81. self._log("Stopping Booker...")
  82. self.m_stop_event.set()
  83. self._cleanup_all_tasks("booker stop")
  84. def update_config(self, new_cfg: GroupConfig):
  85. """动态更新配置"""
  86. with self.m_lock:
  87. if self.m_cfg.enable and not new_cfg.enable:
  88. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  89. elif not self.m_cfg.enable and new_cfg.enable:
  90. self._log("Config dynamically updated: Group ENABLED.")
  91. else:
  92. self._log("Config dynamically updated: Parameters refreshed.")
  93. self.m_cfg = new_cfg
  94. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  95. try:
  96. if instance:
  97. ref_str = f" for task={task_ref}" if task_ref else ""
  98. self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
  99. instance.cleanup()
  100. except Exception as e:
  101. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  102. def _cleanup_task(self, task: Task, reason: str = ""):
  103. if task and task.instance:
  104. self._cleanup_instance(task.instance, reason, task.task_ref)
  105. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  106. removed = False
  107. with self.m_lock:
  108. if task in self.m_tasks:
  109. self.m_tasks.remove(task)
  110. removed = True
  111. self._on_task_removed(task)
  112. if cleanup and removed:
  113. self._cleanup_task(task, reason)
  114. return removed
  115. def _on_task_removed(self, task: Task):
  116. """子类在 task 被移除时的自定义 Hook(如清理缓存)"""
  117. pass
  118. def _cleanup_all_tasks(self, reason: str = ""):
  119. with self.m_lock:
  120. tasks = list(self.m_tasks)
  121. self.m_tasks.clear()
  122. self._on_all_tasks_cleaned()
  123. for task in tasks:
  124. self._cleanup_task(task, reason)
  125. def _on_all_tasks_cleaned(self):
  126. """子类在所有 task 被清空时的自定义 Hook"""
  127. pass
  128. def _get_redis_key(self, routing_key: str) -> str:
  129. return f"vs:signal:{routing_key}"
  130. def _normalize_error(self, err: Exception) -> BizException:
  131. """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
  132. if isinstance(err, BizException):
  133. return err
  134. err_str = str(err)
  135. if "42901" in err_str or "Rate limited" in err_str:
  136. return RateLimiteddError(message=err_str)
  137. if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
  138. return SessionExpiredOrInvalidError(message=err_str)
  139. if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str:
  140. return PermissionDeniedError(message=err_str)
  141. return BizException(
  142. code=50000,
  143. message=f"Unhandled ({type(err).__name__}): {err_str}",
  144. )
  145. def _dispatch_error_policy(
  146. self,
  147. action: str,
  148. err: Exception,
  149. task: Optional[Task] = None,
  150. plg_cfg: Optional[PluginConfig] = None,
  151. task_id: Optional[int] = None
  152. ) -> None:
  153. """
  154. Booker 错误调度与自愈策略引擎 (Policy Engine):
  155. 基于 (Action, 异常事实类型) 矩阵 + Booker 模式闭环执行自愈动作:
  156. - spawn/login 时 429:判定为账号级限流,锁定账号,更新 Redis 订单/任务追踪器与生成冷却;
  157. - book 时 429:档期提交频控或出口 IP 限制,不锁定账号,移除任务以轮换代理或重试;
  158. - session expired / 401:会话失效,移除任务;
  159. - 403 Forbidden / WAF:判定为 IP 受阻,移除任务以轮换代理;
  160. - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
  161. """
  162. biz_err = self._normalize_error(err)
  163. acc = None
  164. if plg_cfg and plg_cfg.account:
  165. acc = plg_cfg.account
  166. elif task and getattr(task.instance, "config", None) and task.instance.config.account:
  167. acc = task.instance.config.account
  168. # 1. 目标站限流 / 429
  169. if isinstance(biz_err, RateLimiteddError):
  170. if action in ("spawn", "login", "create_session"):
  171. backoff = float(self.m_cfg.login_backoff)
  172. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  173. if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0):
  174. try:
  175. CloudApi.Instance().lock_account(acc.id, int(backoff))
  176. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  177. except Exception as lock_err:
  178. self._log(f"Failed to lock account via cloud API: {lock_err}")
  179. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  180. if effective_task_id is not None:
  181. self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
  182. self.m_spawn_cooldown_until = time.time() + backoff
  183. if task:
  184. self._remove_task(task, "spawn account rate limited")
  185. elif action == "book":
  186. # 预订提交时 429:接口频控或 IP 限制,绝不锁账号,移除实例以更换代理重试
  187. self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
  188. if task:
  189. self._remove_task(task, "booking rate limited")
  190. else:
  191. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  192. if task:
  193. self._remove_task(task, f"{action} rate limited")
  194. # 2. 会话过期 / 401
  195. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  196. if action in ("spawn", "login", "create_session"):
  197. self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
  198. self.m_spawn_cooldown_until = time.time() + 5.0
  199. else:
  200. self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
  201. if task:
  202. self._remove_task(task, f"{action} session expired")
  203. # 3. 权限被拒 / WAF 拦截 / 403
  204. elif isinstance(biz_err, PermissionDeniedError):
  205. proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)
  206. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  207. if task:
  208. self._remove_task(task, f"{action} waf blocked")
  209. if action == "spawn":
  210. self.m_spawn_cooldown_until = time.time() + proxy_cd
  211. # 4. 网络或浏览器超时
  212. elif isinstance(biz_err, NetworkTimeoutError):
  213. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  214. if task:
  215. task.next_run = time.time() + 5.0
  216. # 5. 兜底策略
  217. else:
  218. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  219. if action == "spawn":
  220. self.m_spawn_cooldown_until = time.time() + 10.0
  221. elif task:
  222. task.next_run = time.time() + 10.0
  223. def _is_within_active_hours(self) -> bool:
  224. """判断当前是否在允许创建实例的时间段内"""
  225. start_str = self.m_cfg.active_time_start
  226. end_str = self.m_cfg.active_time_end
  227. current_bj_time = datetime.utcnow().time()
  228. start_time = datetime.strptime(start_str, "%H:%M").time()
  229. end_time = datetime.strptime(end_str, "%H:%M").time()
  230. return start_time <= current_bj_time <= end_time
  231. def _maintain_loop(self):
  232. self._log("Maintain loop started.")
  233. while not self.m_stop_event.is_set():
  234. try:
  235. time.sleep(1.0)
  236. with self.m_lock:
  237. tasks_to_check = list(self.m_tasks)
  238. if not tasks_to_check:
  239. continue
  240. dead_tasks = []
  241. healthy_tasks = []
  242. now = time.time()
  243. for t in tasks_to_check:
  244. if now >= t.next_remote_ping:
  245. t.instance.keep_alive()
  246. if t.instance.health_check():
  247. healthy_tasks.append(t)
  248. t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
  249. else:
  250. dead_tasks.append(t)
  251. else:
  252. healthy_tasks.append(t)
  253. self._on_maintain_ping(healthy_tasks, dead_tasks)
  254. if dead_tasks:
  255. with self.m_lock:
  256. current_tasks = list(self.m_tasks)
  257. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  258. for t in dead_tasks:
  259. if t in current_tasks:
  260. self._cleanup_task(t, "unhealthy or keep-alive failed")
  261. else:
  262. with self.m_lock:
  263. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  264. except Exception as e:
  265. self._log(f'Maintain loop exception: {e}')
  266. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  267. """子类维护循环中对于健康/异常 Task 的额外处理"""
  268. pass
  269. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  270. """判断 query_result 中的可用日期是否在 task 意向范围内(默认 True,Order 模式重写)"""
  271. return True
  272. def _booking_trigger_loop(self):
  273. self._log("Trigger loop started.")
  274. while not self.m_stop_event.is_set():
  275. try:
  276. time.sleep(0.1)
  277. now = time.time()
  278. for apt_type in self.m_cfg.appointment_types:
  279. redis_key = self._get_redis_key(apt_type.routing_key)
  280. raw_data = self.redis_client.get(redis_key)
  281. if not raw_data:
  282. continue
  283. try:
  284. data = json.loads(raw_data)
  285. query_result = QueryResult.model_validate(data['query_result'])
  286. query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
  287. except Exception as parse_err:
  288. self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
  289. self.redis_client.delete(redis_key)
  290. continue
  291. matching_tasks = []
  292. with self.m_lock:
  293. for task in self.m_tasks:
  294. if now < task.next_run or not task.book_allowed:
  295. continue
  296. if apt_type.routing_key not in task.acceptable_routing_keys:
  297. continue
  298. if not self._is_date_of_interest(task, query_result):
  299. continue
  300. task.next_run = now + self.m_cfg.booker.booking_cooldown
  301. matching_tasks.append(task)
  302. if matching_tasks:
  303. threads = []
  304. for task in matching_tasks:
  305. self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
  306. t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
  307. threads.append(t)
  308. t.start()
  309. for t in threads:
  310. t.join()
  311. except Exception as e:
  312. self._log(f"Booking trigger loop exception: {e}")
  313. def _execute_book_job(self, task: Task, query_result: QueryResult):
  314. raise NotImplementedError
  315. def _creator_loop(self):
  316. raise NotImplementedError
  317. def _push_success_notification(self, task_id, order_id, book_res):
  318. """成功落单后同步云端与推送微信通知的公共方法"""
  319. grab_info = {
  320. "account": book_res.account,
  321. "session_id": book_res.session_id,
  322. "urn": book_res.urn,
  323. "slot_date": book_res.book_date,
  324. "slot_time": book_res.book_time,
  325. "timestamp": int(time.time()),
  326. "payment_link": book_res.payment_link
  327. }
  328. def _update_cloud_success():
  329. try:
  330. CloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
  331. except Exception as e:
  332. self._log(f"Failed to update success state to cloud: {e}")
  333. ThreadPool.getInstance().enqueue(_update_cloud_success)
  334. notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
  335. self.redis_client.zrem(self.m_tracker_key, task_id)
  336. class BuiltinBookerGCO(BaseBookerGCO):
  337. """
  338. 非绑定模式 (公共内置账号池):
  339. - 只维护全局 target_instances 数量的实例。
  340. - 所有实例热机等待,发现信号后临时去云端 Pop 订单。
  341. """
  342. TAG = "BUILTIN-BOOKER"
  343. def _creator_loop(self):
  344. self._log("Creator loop started.")
  345. while not self.m_stop_event.wait(1.0):
  346. if not self._is_within_active_hours():
  347. continue
  348. if time.time() < self.m_spawn_cooldown_until:
  349. continue
  350. with self.m_lock:
  351. current = len(self.m_tasks)
  352. target = self.m_cfg.booker.target_instances
  353. if current < target:
  354. self._spawn_worker()
  355. def _spawn_worker(self):
  356. instance = None
  357. task = None
  358. success = False
  359. plg_cfg = None
  360. try:
  361. plg_cfg = PluginConfig()
  362. plg_cfg.debug = self.m_cfg.debug
  363. plg_cfg.free_config = self.m_cfg.free_config
  364. plg_cfg.session_max_life = self.m_cfg.session_max_life
  365. if self.m_cfg.need_account:
  366. acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  367. plg_cfg.account = type(plg_cfg.account)(**acc)
  368. if self.m_cfg.need_proxy:
  369. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  370. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  371. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  372. instance.set_log(self.m_logger)
  373. instance.set_config(plg_cfg)
  374. instance.create_session()
  375. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  376. task = Task(
  377. instance=instance,
  378. next_run=time.time(),
  379. task_ref=None,
  380. acceptable_routing_keys=all_keys,
  381. source_queue="built-in",
  382. book_allowed=True,
  383. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  384. )
  385. with self.m_lock:
  386. self.m_tasks.append(task)
  387. success = True
  388. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  389. except Exception as e:
  390. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  391. finally:
  392. if not success and instance:
  393. self._cleanup_instance(instance, "spawn failed")
  394. def _execute_book_job(self, task: Task, query_result: QueryResult):
  395. queue_name = f"auto.{query_result.apt_type.routing_key}"
  396. task_id = None
  397. task_data = None
  398. try:
  399. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  400. if not task_data:
  401. return
  402. task_id = task_data['id']
  403. order_id = task_data.get('order_id')
  404. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
  405. user_input = task_data.get('user_inputs', {})
  406. book_res = task.instance.book(query_result, user_input)
  407. if book_res.success:
  408. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  409. self._push_success_notification(task_id, order_id, book_res)
  410. task.successful_bookings += 1
  411. max_b = self.m_cfg.booker.max_bookings_per_account
  412. if max_b > 0 and task.successful_bookings >= max_b:
  413. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  414. self._remove_task(task, "max bookings reached")
  415. else:
  416. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  417. except Exception as e:
  418. self._dispatch_error_policy("book", e, task=task)
  419. class OrderBookerGCO(BaseBookerGCO):
  420. """
  421. 绑定模式 (订单自带账号):
  422. - 按城市队列维护热机配额。
  423. - 绝对的 1 对 1 关系:一个实例绑定一个云端订单。
  424. - 预订成功后,实例立即销毁。
  425. """
  426. TAG = "ORDER-BOOKER"
  427. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  428. super().__init__(cfg, redis_conf, logger)
  429. self.m_task_data_cache: Dict[str, dict] = {}
  430. self.heartbeat_ttl = 2 * 60.0
  431. def _start_additional_threads(self):
  432. threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
  433. def _on_task_removed(self, task: Task):
  434. task_id = task.task_ref
  435. if task_id:
  436. self.m_task_data_cache.pop(str(task_id), None)
  437. def _on_all_tasks_cleaned(self):
  438. self.m_task_data_cache.clear()
  439. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  440. if healthy_tasks:
  441. new_deadline = time.time() + self.heartbeat_ttl
  442. mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
  443. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  444. if dead_tasks:
  445. mapping = {str(t.task_ref): 0 for t in dead_tasks}
  446. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  447. def _cache_refresh_loop(self):
  448. self._log("Cache refresh loop started.")
  449. refresh_interval = 15 * 60
  450. while not self.m_stop_event.is_set():
  451. try:
  452. time.sleep(1)
  453. with self.m_lock:
  454. tasks_to_check = {
  455. tid: data.get('_last_refresh', 0)
  456. for tid, data in self.m_task_data_cache.items()
  457. }
  458. if not tasks_to_check:
  459. continue
  460. now = time.time()
  461. for tid, last_refresh in tasks_to_check.items():
  462. if now - last_refresh >= refresh_interval:
  463. fresh_data = CloudApi.Instance().get_vas_task(tid)
  464. if fresh_data:
  465. fresh_data['_last_refresh'] = time.time()
  466. with self.m_lock:
  467. if tid in self.m_task_data_cache:
  468. self.m_task_data_cache[tid] = fresh_data
  469. time.sleep(0.5)
  470. except Exception as e:
  471. self._log(f'Cache refresh loop exception: {e}')
  472. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  473. if query_result.availability_status != AvailabilityStatus.Available:
  474. return True
  475. task_id = task.task_ref
  476. task_data = self.m_task_data_cache.get(str(task_id), {})
  477. user_input = task_data.get('user_inputs', {})
  478. expected_end_date = (
  479. user_input.get('expected_end_date')
  480. or '2100-01-01'
  481. )
  482. available_date = query_result.earliest_date
  483. dt = available_date.strftime("%Y-%m-%d")
  484. return dt <= expected_end_date
  485. def _execute_book_job(self, task: Task, query_result: QueryResult):
  486. task_id = task.task_ref
  487. task_data = None
  488. try:
  489. with self.m_lock:
  490. task_data = self.m_task_data_cache.get(str(task_id))
  491. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
  492. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  493. self._remove_task(task, "bound task no longer valid")
  494. self.redis_client.zrem(self.m_tracker_key, task_id)
  495. return
  496. order_id = task_data.get('order_id')
  497. user_input = task_data.get('user_inputs', {})
  498. book_res = task.instance.book(query_result, user_input)
  499. if book_res.success:
  500. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  501. self._push_success_notification(task_id, order_id, book_res)
  502. self._remove_task(task, "booking success")
  503. else:
  504. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  505. except Exception as e:
  506. self._dispatch_error_policy("book", e, task=task)
  507. def _creator_loop(self):
  508. self._log("Creator loop started.")
  509. while not self.m_stop_event.wait(1.0):
  510. if not self._is_within_active_hours():
  511. continue
  512. if time.time() < self.m_spawn_cooldown_until:
  513. continue
  514. for apt in self.m_cfg.appointment_types:
  515. r_key = apt.routing_key
  516. with self.m_lock:
  517. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  518. target = self.m_cfg.booker.target_instances
  519. if active < target:
  520. self._spawn_worker(r_key)
  521. def _spawn_worker(self, target_routing_key: str):
  522. instance = None
  523. task = None
  524. success = False
  525. task_id = None
  526. try:
  527. queue_name = f"auto.{target_routing_key}"
  528. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  529. if not task_data:
  530. return
  531. task_id = task_data['id']
  532. with self.m_lock:
  533. self.m_task_data_cache[str(task_id)] = task_data
  534. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
  535. user_inputs = task_data.get('user_inputs', {})
  536. plg_cfg = PluginConfig()
  537. plg_cfg.debug = self.m_cfg.debug
  538. plg_cfg.free_config = self.m_cfg.free_config
  539. plg_cfg.session_max_life = self.m_cfg.session_max_life
  540. plg_cfg.account.username = user_inputs.get("username", "")
  541. plg_cfg.account.password = user_inputs.get("password", "")
  542. if not plg_cfg.account.username:
  543. return
  544. acceptable_keys = [target_routing_key]
  545. if self.m_cfg.need_proxy:
  546. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  547. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  548. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  549. instance.set_log(self.m_logger)
  550. instance.set_config(plg_cfg)
  551. instance.create_session()
  552. task = Task(
  553. instance=instance,
  554. next_run=time.time(),
  555. task_ref=task_id,
  556. acceptable_routing_keys=acceptable_keys,
  557. source_queue=target_routing_key,
  558. book_allowed=True,
  559. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  560. )
  561. with self.m_lock:
  562. self.m_tasks.append(task)
  563. success = True
  564. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  565. except Exception as e:
  566. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
  567. finally:
  568. if not success and instance:
  569. self._cleanup_instance(instance, "spawn failed")
  570. if not success and task_id:
  571. with self.m_lock:
  572. self.m_task_data_cache.pop(str(task_id), None)