booker.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. import os
  2. import time
  3. import json
  4. import threading
  5. import random
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from core_types import (
  9. GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult,
  10. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
  11. BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
  12. NetworkTimeoutError
  13. )
  14. from plugin_factory import PluginFactory
  15. from toolkit.cloud_api import CloudApi
  16. from toolkit.proxy_manager import ProxyManager
  17. from toolkit.thread_pool import ThreadPool
  18. from toolkit.notifier import notify_book_result
  19. from utils.safe_redis_cli import SafeRedisClient
  20. class BaseBookerGCO:
  21. """
  22. Booker 基类,封装公共的基础设施与通用的生命周期管理、逻辑循环等。
  23. """
  24. TAG = "BOOKER"
  25. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  26. self.m_cfg = cfg
  27. self.m_factory = PluginFactory()
  28. self.m_logger = logger
  29. self.m_tasks: List[Task] = []
  30. self.m_lock = threading.RLock()
  31. self.m_stop_event = threading.Event()
  32. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  33. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  34. self.m_last_group_query_time = 0.0
  35. self.m_spawn_cooldown_until = 0.0
  36. def _log(self, message):
  37. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  38. if self.m_logger:
  39. self.m_logger(f'{prefix} {message}')
  40. else:
  41. print(f'{prefix} {message}')
  42. def get_status(self) -> GroupStatus:
  43. with self.m_lock:
  44. tasks = list(self.m_tasks)
  45. gid = self.m_cfg.identifier
  46. enable = self.m_cfg.enable
  47. plugin_name = self.m_cfg.plugin_config.plugin_name
  48. plugins_status = []
  49. healthy_count = 0
  50. for t in tasks:
  51. if t and t.instance:
  52. st = t.instance.get_plugin_status()
  53. if st.is_healthy:
  54. healthy_count += 1
  55. plugins_status.append(st)
  56. return GroupStatus(
  57. group_id=gid,
  58. enable=enable,
  59. plugin_name=plugin_name,
  60. total_tasks=len(tasks),
  61. healthy_tasks=healthy_count,
  62. plugins=plugins_status,
  63. updated_at=utc_now_str(),
  64. )
  65. def start(self):
  66. if not self.m_cfg.enable:
  67. return
  68. self._log(f"Starting {self.TAG}...")
  69. plugin_name = self.m_cfg.plugin_config.plugin_name
  70. class_name = "".join(part.title() for part in plugin_name.split('_'))
  71. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  72. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  73. threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
  74. threading.Thread(target=self._creator_loop, daemon=True).start()
  75. threading.Thread(target=self._maintain_loop, daemon=True).start()
  76. self._start_additional_threads()
  77. def _start_additional_threads(self):
  78. """子类扩展线程的 Hook"""
  79. pass
  80. def stop(self):
  81. self._log("Stopping Booker...")
  82. self.m_stop_event.set()
  83. self._cleanup_all_tasks("booker stop")
  84. def update_config(self, new_cfg: GroupConfig):
  85. """动态更新配置"""
  86. with self.m_lock:
  87. if self.m_cfg.enable and not new_cfg.enable:
  88. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  89. elif not self.m_cfg.enable and new_cfg.enable:
  90. self._log("Config dynamically updated: Group ENABLED.")
  91. else:
  92. self._log("Config dynamically updated: Parameters refreshed.")
  93. self.m_cfg = new_cfg
  94. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  95. try:
  96. if instance:
  97. ref_str = f" for task={task_ref}" if task_ref else ""
  98. self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
  99. instance.cleanup()
  100. except Exception as e:
  101. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  102. def _cleanup_task(self, task: Task, reason: str = ""):
  103. if task and task.instance:
  104. self._cleanup_instance(task.instance, reason, task.task_ref)
  105. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  106. removed = False
  107. with self.m_lock:
  108. if task in self.m_tasks:
  109. self.m_tasks.remove(task)
  110. removed = True
  111. self._on_task_removed(task)
  112. if cleanup and removed:
  113. self._cleanup_task(task, reason)
  114. return removed
  115. def _on_task_removed(self, task: Task):
  116. """子类在 task 被移除时的自定义 Hook(如清理缓存)"""
  117. pass
  118. def _cleanup_all_tasks(self, reason: str = ""):
  119. with self.m_lock:
  120. tasks = list(self.m_tasks)
  121. self.m_tasks.clear()
  122. self._on_all_tasks_cleaned()
  123. for task in tasks:
  124. self._cleanup_task(task, reason)
  125. def _on_all_tasks_cleaned(self):
  126. """子类在所有 task 被清空时的自定义 Hook"""
  127. pass
  128. def _get_redis_key(self, routing_key: str) -> str:
  129. return f"vs:signal:{routing_key}"
  130. def _normalize_error(self, err: Exception) -> BizException:
  131. """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
  132. err_str = str(err)
  133. # 如果是底层网络/CORS预检被拒导致的 Fetch 失败或 WAF 阻断,精准提升为 PermissionDeniedError
  134. if "Failed to fetch" in err_str or "Status: 0" in err_str or "WAF Blocked" in err_str or "cf-mitigated" in err_str:
  135. return PermissionDeniedError(message=err_str)
  136. if isinstance(err, BizException):
  137. return err
  138. if "42901" in err_str or "Rate limited" in err_str:
  139. return RateLimiteddError(message=err_str)
  140. if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
  141. return SessionExpiredOrInvalidError(message=err_str)
  142. if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str:
  143. return PermissionDeniedError(message=err_str)
  144. return BizException(
  145. code=50000,
  146. message=f"Unhandled ({type(err).__name__}): {err_str}",
  147. )
  148. def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
  149. """安全归还账号:将云端冷却时间重置为 0,避免非账号问题导致账号被误锁 3 小时"""
  150. if acc and acc.id > 0 and acc.username not in ("", "Guest"):
  151. try:
  152. CloudApi.Instance().lock_account(acc.id, 0)
  153. self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
  154. except Exception as e:
  155. self._log(f"Failed to release account [{acc.username}]: {e}")
  156. def _dispatch_error_policy(
  157. self,
  158. action: str,
  159. err: Exception,
  160. task: Optional[Task] = None,
  161. plg_cfg: Optional[PluginConfig] = None,
  162. task_id: Optional[int] = None
  163. ) -> None:
  164. """
  165. Booker 错误调度与自愈策略引擎 (Policy Engine):
  166. 基于 (Action, 异常事实类型) 矩阵 + Booker 模式闭环执行自愈动作:
  167. - spawn/login 时 429:判定为账号级限流,锁定账号,更新 Redis 订单/任务追踪器与生成冷却;
  168. - book 时 429:档期提交频控或出口 IP 限制,不锁定账号,移除任务以轮换代理或重试;
  169. - session expired / 401:会话失效,移除任务;
  170. - 403 Forbidden / WAF:判定为 IP 受阻,移除任务以轮换代理;在 spawn 时立即安全释放账号;
  171. - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
  172. """
  173. biz_err = self._normalize_error(err)
  174. active_plg_cfg = plg_cfg or (task.instance.config if task else None)
  175. acc = active_plg_cfg.account if active_plg_cfg else None
  176. # 1. 目标站限流 / 429
  177. if isinstance(biz_err, RateLimiteddError):
  178. if action in ("spawn", "login", "create_session"):
  179. backoff = float(self.m_cfg.login_backoff)
  180. self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
  181. if acc and acc.username not in ("", "Guest") and acc.id > 0:
  182. try:
  183. CloudApi.Instance().lock_account(acc.id, int(backoff))
  184. self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
  185. except Exception as lock_err:
  186. self._log(f"Failed to lock account via cloud API: {lock_err}")
  187. effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
  188. if effective_task_id is not None:
  189. self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
  190. self.m_spawn_cooldown_until = time.time() + backoff
  191. if task:
  192. self._remove_task(task, "spawn account rate limited")
  193. elif action == "book":
  194. # 预订提交时 429:接口频控或 IP 限制,绝不锁账号,移除实例以更换代理重试
  195. self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
  196. if task:
  197. self._remove_task(task, "booking rate limited")
  198. else:
  199. self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
  200. if task:
  201. self._remove_task(task, f"{action} rate limited")
  202. # 2. 会话过期 / 401
  203. elif isinstance(biz_err, SessionExpiredOrInvalidError):
  204. if action in ("spawn", "login", "create_session"):
  205. self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
  206. self.m_spawn_cooldown_until = time.time() + 5.0
  207. else:
  208. self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
  209. if task:
  210. self._remove_task(task, f"{action} session expired")
  211. # 3. 权限被拒 / WAF 拦截 / 403
  212. elif isinstance(biz_err, PermissionDeniedError):
  213. proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
  214. self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
  215. if task:
  216. self._remove_task(task, f"{action} waf blocked")
  217. if action in ("spawn", "login", "create_session"):
  218. self._release_account_safely(acc, reason="WAF Blocked")
  219. self.m_spawn_cooldown_until = time.time() + proxy_cd
  220. # 4. 网络或浏览器超时
  221. elif isinstance(biz_err, NetworkTimeoutError):
  222. self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
  223. if task:
  224. task.next_run = time.time() + 5.0
  225. if action in ("spawn", "login", "create_session"):
  226. self._release_account_safely(acc, reason="Network Timeout")
  227. self.m_spawn_cooldown_until = time.time() + 10.0
  228. # 5. 兜底策略
  229. else:
  230. self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
  231. if action in ("spawn", "login", "create_session"):
  232. self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
  233. self.m_spawn_cooldown_until = time.time() + 10.0
  234. elif task:
  235. task.next_run = time.time() + 10.0
  236. # 6. 统一代理健康指标埋点
  237. if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
  238. err_type = biz_err.message
  239. if isinstance(biz_err, PermissionDeniedError):
  240. if "403202" in err_type or "cf-mitigated" in err_type.lower() or "waf blocked" in err_type.lower():
  241. err_type = "cf-mitigated (403202)"
  242. ProxyManager.Instance().mark_proxy_result(
  243. proxy=active_plg_cfg.proxy.model_dump(),
  244. plugin_name=self.m_cfg.plugin_config.plugin_name,
  245. action=action,
  246. success=False,
  247. http_code=biz_err.http_status,
  248. error_type=err_type
  249. )
  250. def _is_within_active_hours(self) -> bool:
  251. """判断当前是否在允许创建实例的时间段内"""
  252. start_str = self.m_cfg.active_time_start
  253. end_str = self.m_cfg.active_time_end
  254. current_bj_time = datetime.utcnow().time()
  255. start_time = datetime.strptime(start_str, "%H:%M").time()
  256. end_time = datetime.strptime(end_str, "%H:%M").time()
  257. return start_time <= current_bj_time <= end_time
  258. def _maintain_loop(self):
  259. self._log("Maintain loop started.")
  260. while not self.m_stop_event.is_set():
  261. try:
  262. time.sleep(1.0)
  263. with self.m_lock:
  264. tasks_to_check = list(self.m_tasks)
  265. if not tasks_to_check:
  266. continue
  267. dead_tasks = []
  268. healthy_tasks = []
  269. now = time.time()
  270. for t in tasks_to_check:
  271. if now >= t.next_remote_ping:
  272. t.instance.keep_alive()
  273. if t.instance.health_check():
  274. healthy_tasks.append(t)
  275. t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
  276. else:
  277. dead_tasks.append(t)
  278. else:
  279. healthy_tasks.append(t)
  280. self._on_maintain_ping(healthy_tasks, dead_tasks)
  281. if dead_tasks:
  282. with self.m_lock:
  283. current_tasks = list(self.m_tasks)
  284. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  285. for t in dead_tasks:
  286. if t in current_tasks:
  287. self._cleanup_task(t, "unhealthy or keep-alive failed")
  288. else:
  289. with self.m_lock:
  290. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  291. except Exception as e:
  292. self._log(f'Maintain loop exception: {e}')
  293. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  294. """子类维护循环中对于健康/异常 Task 的额外处理"""
  295. pass
  296. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  297. """判断 query_result 中的可用日期是否在 task 意向范围内(默认 True,Order 模式重写)"""
  298. return True
  299. def _booking_trigger_loop(self):
  300. self._log("Trigger loop started.")
  301. while not self.m_stop_event.is_set():
  302. try:
  303. time.sleep(0.1)
  304. now = time.time()
  305. for apt_type in self.m_cfg.appointment_types:
  306. redis_key = self._get_redis_key(apt_type.routing_key)
  307. raw_data = self.redis_client.get(redis_key)
  308. if not raw_data:
  309. continue
  310. try:
  311. data = json.loads(raw_data)
  312. query_result = QueryResult.model_validate(data['query_result'])
  313. query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
  314. except Exception as parse_err:
  315. self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
  316. self.redis_client.delete(redis_key)
  317. continue
  318. matching_tasks = []
  319. with self.m_lock:
  320. for task in self.m_tasks:
  321. if now < task.next_run or not task.book_allowed:
  322. continue
  323. if apt_type.routing_key not in task.acceptable_routing_keys:
  324. continue
  325. if not self._is_date_of_interest(task, query_result):
  326. continue
  327. task.next_run = now + self.m_cfg.booker.booking_cooldown
  328. matching_tasks.append(task)
  329. if matching_tasks:
  330. threads = []
  331. for task in matching_tasks:
  332. self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
  333. t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
  334. threads.append(t)
  335. t.start()
  336. for t in threads:
  337. t.join()
  338. except Exception as e:
  339. self._log(f"Booking trigger loop exception: {e}")
  340. def _execute_book_job(self, task: Task, query_result: QueryResult):
  341. raise NotImplementedError
  342. def _creator_loop(self):
  343. raise NotImplementedError
  344. def _push_success_notification(self, task_id, order_id, book_res):
  345. """成功落单后同步云端与推送微信通知的公共方法"""
  346. grab_info = {
  347. "account": book_res.account,
  348. "session_id": book_res.session_id,
  349. "urn": book_res.urn,
  350. "slot_date": book_res.book_date,
  351. "slot_time": book_res.book_time,
  352. "timestamp": int(time.time()),
  353. "payment_link": book_res.payment_link
  354. }
  355. def _update_cloud_success():
  356. try:
  357. CloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
  358. except Exception as e:
  359. self._log(f"Failed to update success state to cloud: {e}")
  360. ThreadPool.getInstance().enqueue(_update_cloud_success)
  361. notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
  362. self.redis_client.zrem(self.m_tracker_key, task_id)
  363. class BuiltinBookerGCO(BaseBookerGCO):
  364. """
  365. 非绑定模式 (公共内置账号池):
  366. - 只维护全局 target_instances 数量的实例。
  367. - 所有实例热机等待,发现信号后临时去云端 Pop 订单。
  368. """
  369. TAG = "BUILTIN-BOOKER"
  370. def _creator_loop(self):
  371. self._log("Creator loop started.")
  372. while not self.m_stop_event.wait(1.0):
  373. if not self._is_within_active_hours():
  374. continue
  375. if time.time() < self.m_spawn_cooldown_until:
  376. continue
  377. with self.m_lock:
  378. current = len(self.m_tasks)
  379. target = self.m_cfg.booker.target_instances
  380. if current < target:
  381. self._spawn_worker()
  382. def _spawn_worker(self):
  383. instance = None
  384. task = None
  385. success = False
  386. plg_cfg = None
  387. try:
  388. plg_cfg = PluginConfig()
  389. plg_cfg.debug = self.m_cfg.debug
  390. plg_cfg.free_config = self.m_cfg.free_config
  391. plg_cfg.session_max_life = self.m_cfg.session_max_life
  392. if self.m_cfg.need_account:
  393. acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  394. plg_cfg.account = type(plg_cfg.account)(**acc)
  395. if self.m_cfg.need_proxy:
  396. proxy = ProxyManager.Instance().get_next_proxy(
  397. self.m_cfg.proxy_pool,
  398. self.m_cfg.proxy_cd,
  399. plugin_name=self.m_cfg.plugin_config.plugin_name
  400. )
  401. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  402. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  403. instance.set_log(self.m_logger)
  404. instance.set_config(plg_cfg)
  405. instance.create_session()
  406. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  407. task = Task(
  408. instance=instance,
  409. next_run=time.time(),
  410. task_ref=None,
  411. acceptable_routing_keys=all_keys,
  412. source_queue="built-in",
  413. book_allowed=True,
  414. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  415. )
  416. with self.m_lock:
  417. self.m_tasks.append(task)
  418. success = True
  419. if plg_cfg.proxy.port > 0:
  420. ProxyManager.Instance().mark_proxy_result(
  421. proxy=plg_cfg.proxy.model_dump(),
  422. plugin_name=self.m_cfg.plugin_config.plugin_name,
  423. action="spawn",
  424. success=True,
  425. http_code=200
  426. )
  427. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  428. except Exception as e:
  429. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
  430. finally:
  431. if not success and instance:
  432. self._cleanup_instance(instance, "spawn failed")
  433. def _execute_book_job(self, task: Task, query_result: QueryResult):
  434. queue_name = f"auto.{query_result.apt_type.routing_key}"
  435. task_id = None
  436. task_data = None
  437. try:
  438. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  439. if not task_data:
  440. return
  441. task_id = task_data['id']
  442. order_id = task_data.get('order_id')
  443. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
  444. user_input = task_data.get('user_inputs', {})
  445. book_res = task.instance.book(query_result, user_input)
  446. if book_res.success:
  447. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  448. self._push_success_notification(task_id, order_id, book_res)
  449. task.successful_bookings += 1
  450. max_b = self.m_cfg.booker.max_bookings_per_account
  451. if max_b > 0 and task.successful_bookings >= max_b:
  452. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  453. self._remove_task(task, "max bookings reached")
  454. else:
  455. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  456. except Exception as e:
  457. self._dispatch_error_policy("book", e, task=task)
  458. class OrderBookerGCO(BaseBookerGCO):
  459. """
  460. 绑定模式 (订单自带账号):
  461. - 按城市队列维护热机配额。
  462. - 绝对的 1 对 1 关系:一个实例绑定一个云端订单。
  463. - 预订成功后,实例立即销毁。
  464. """
  465. TAG = "ORDER-BOOKER"
  466. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  467. super().__init__(cfg, redis_conf, logger)
  468. self.m_task_data_cache: Dict[str, dict] = {}
  469. self.heartbeat_ttl = 2 * 60.0
  470. def _start_additional_threads(self):
  471. threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
  472. def _on_task_removed(self, task: Task):
  473. task_id = task.task_ref
  474. if task_id:
  475. self.m_task_data_cache.pop(str(task_id), None)
  476. def _on_all_tasks_cleaned(self):
  477. self.m_task_data_cache.clear()
  478. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  479. if healthy_tasks:
  480. new_deadline = time.time() + self.heartbeat_ttl
  481. mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
  482. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  483. if dead_tasks:
  484. mapping = {str(t.task_ref): 0 for t in dead_tasks}
  485. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  486. def _cache_refresh_loop(self):
  487. self._log("Cache refresh loop started.")
  488. refresh_interval = 15 * 60
  489. while not self.m_stop_event.is_set():
  490. try:
  491. time.sleep(1)
  492. with self.m_lock:
  493. tasks_to_check = {
  494. tid: data.get('_last_refresh', 0)
  495. for tid, data in self.m_task_data_cache.items()
  496. }
  497. if not tasks_to_check:
  498. continue
  499. now = time.time()
  500. for tid, last_refresh in tasks_to_check.items():
  501. if now - last_refresh >= refresh_interval:
  502. fresh_data = CloudApi.Instance().get_vas_task(tid)
  503. if fresh_data:
  504. fresh_data['_last_refresh'] = time.time()
  505. with self.m_lock:
  506. if tid in self.m_task_data_cache:
  507. self.m_task_data_cache[tid] = fresh_data
  508. time.sleep(0.5)
  509. except Exception as e:
  510. self._log(f'Cache refresh loop exception: {e}')
  511. def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
  512. if query_result.availability_status != AvailabilityStatus.Available:
  513. return True
  514. task_id = task.task_ref
  515. task_data = self.m_task_data_cache.get(str(task_id), {})
  516. user_input = task_data.get('user_inputs', {})
  517. expected_end_date = (
  518. user_input.get('expected_end_date')
  519. or '2100-01-01'
  520. )
  521. available_date = query_result.earliest_date
  522. dt = available_date.strftime("%Y-%m-%d")
  523. return dt <= expected_end_date
  524. def _execute_book_job(self, task: Task, query_result: QueryResult):
  525. task_id = task.task_ref
  526. task_data = None
  527. try:
  528. with self.m_lock:
  529. task_data = self.m_task_data_cache.get(str(task_id))
  530. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
  531. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  532. self._remove_task(task, "bound task no longer valid")
  533. self.redis_client.zrem(self.m_tracker_key, task_id)
  534. return
  535. order_id = task_data.get('order_id')
  536. user_input = task_data.get('user_inputs', {})
  537. book_res = task.instance.book(query_result, user_input)
  538. if book_res.success:
  539. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  540. self._push_success_notification(task_id, order_id, book_res)
  541. self._remove_task(task, "booking success")
  542. else:
  543. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  544. except Exception as e:
  545. self._dispatch_error_policy("book", e, task=task)
  546. def _creator_loop(self):
  547. self._log("Creator loop started.")
  548. while not self.m_stop_event.wait(1.0):
  549. if not self._is_within_active_hours():
  550. continue
  551. if time.time() < self.m_spawn_cooldown_until:
  552. continue
  553. for apt in self.m_cfg.appointment_types:
  554. r_key = apt.routing_key
  555. with self.m_lock:
  556. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  557. target = self.m_cfg.booker.target_instances
  558. if active < target:
  559. self._spawn_worker(r_key)
  560. def _spawn_worker(self, target_routing_key: str):
  561. instance = None
  562. task = None
  563. success = False
  564. task_id = None
  565. plg_cfg = None
  566. try:
  567. queue_name = f"auto.{target_routing_key}"
  568. task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
  569. if not task_data:
  570. return
  571. task_id = task_data['id']
  572. with self.m_lock:
  573. self.m_task_data_cache[str(task_id)] = task_data
  574. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
  575. user_inputs = task_data.get('user_inputs', {})
  576. plg_cfg = PluginConfig()
  577. plg_cfg.debug = self.m_cfg.debug
  578. plg_cfg.free_config = self.m_cfg.free_config
  579. plg_cfg.session_max_life = self.m_cfg.session_max_life
  580. plg_cfg.account.username = user_inputs.get("username", "")
  581. plg_cfg.account.password = user_inputs.get("password", "")
  582. if not plg_cfg.account.username:
  583. return
  584. acceptable_keys = [target_routing_key]
  585. if self.m_cfg.need_proxy:
  586. proxy = ProxyManager.Instance().get_next_proxy(
  587. self.m_cfg.proxy_pool,
  588. self.m_cfg.proxy_cd,
  589. plugin_name=self.m_cfg.plugin_config.plugin_name
  590. )
  591. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  592. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  593. instance.set_log(self.m_logger)
  594. instance.set_config(plg_cfg)
  595. instance.create_session()
  596. task = Task(
  597. instance=instance,
  598. next_run=time.time(),
  599. task_ref=task_id,
  600. acceptable_routing_keys=acceptable_keys,
  601. source_queue=target_routing_key,
  602. book_allowed=True,
  603. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  604. )
  605. with self.m_lock:
  606. self.m_tasks.append(task)
  607. success = True
  608. if plg_cfg.proxy.port > 0:
  609. ProxyManager.Instance().mark_proxy_result(
  610. proxy=plg_cfg.proxy.model_dump(),
  611. plugin_name=self.m_cfg.plugin_config.plugin_name,
  612. action="spawn",
  613. success=True,
  614. http_code=200
  615. )
  616. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  617. except Exception as e:
  618. self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
  619. finally:
  620. if not success and instance:
  621. self._cleanup_instance(instance, "spawn failed")
  622. if not success and task_id:
  623. with self.m_lock:
  624. self.m_task_data_cache.pop(str(task_id), None)