booker.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620
  1. import os
  2. import time
  3. import json
  4. import threading
  5. import random
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from vs_types import GroupConfig, VSPlgConfig, Task, VSQueryResult, AppointmentType, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str
  9. from vs_plg_factory import VSPlgFactory
  10. from toolkit.thread_pool import ThreadPool
  11. from toolkit.vs_cloud_api import VSCloudApi
  12. from utils.safe_redis_cli import SafeRedisClient
  13. class BaseBookerGCO:
  14. """
  15. Booker 基类,封装公共的基础设施与通用的生命周期管理、逻辑循环等。
  16. """
  17. TAG = "BOOKER"
  18. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  19. self.m_cfg = cfg
  20. self.m_factory = VSPlgFactory()
  21. self.m_logger = logger
  22. self.m_tasks: List[Task] = []
  23. self.m_lock = threading.RLock()
  24. self.m_stop_event = threading.Event()
  25. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  26. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  27. self.m_last_group_query_time = 0.0
  28. self.m_spawn_cooldown_until = 0.0
  29. def _log(self, message):
  30. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  31. if self.m_logger:
  32. self.m_logger(f'{prefix} {message}')
  33. else:
  34. print(f'{prefix} {message}')
  35. def get_status(self) -> GroupStatus:
  36. with self.m_lock:
  37. tasks = list(self.m_tasks)
  38. gid = self.m_cfg.identifier
  39. enable = self.m_cfg.enable
  40. plugin_name = self.m_cfg.plugin_config.plugin_name
  41. plugins_status = []
  42. healthy_count = 0
  43. for t in tasks:
  44. if t and t.instance:
  45. st = t.instance.get_plugin_status()
  46. if st.is_healthy:
  47. healthy_count += 1
  48. plugins_status.append(st)
  49. return GroupStatus(
  50. group_id=gid,
  51. enable=enable,
  52. plugin_name=plugin_name,
  53. total_tasks=len(tasks),
  54. healthy_tasks=healthy_count,
  55. plugins=plugins_status,
  56. updated_at=utc_now_str(),
  57. )
  58. def start(self):
  59. if not self.m_cfg.enable:
  60. return
  61. self._log(f"Starting {self.TAG}...")
  62. plugin_name = self.m_cfg.plugin_config.plugin_name
  63. class_name = "".join(part.title() for part in plugin_name.split('_'))
  64. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  65. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  66. threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
  67. threading.Thread(target=self._creator_loop, daemon=True).start()
  68. threading.Thread(target=self._maintain_loop, daemon=True).start()
  69. self._start_additional_threads()
  70. def _start_additional_threads(self):
  71. """子类扩展线程的 Hook"""
  72. pass
  73. def stop(self):
  74. self._log("Stopping Booker...")
  75. self.m_stop_event.set()
  76. self._cleanup_all_tasks("booker stop")
  77. def update_config(self, new_cfg: GroupConfig):
  78. """动态更新配置"""
  79. with self.m_lock:
  80. if self.m_cfg.enable and not new_cfg.enable:
  81. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  82. elif not self.m_cfg.enable and new_cfg.enable:
  83. self._log("Config dynamically updated: Group ENABLED.")
  84. else:
  85. self._log("Config dynamically updated: Parameters refreshed.")
  86. self.m_cfg = new_cfg
  87. def _cleanup_task(self, task: Task, reason: str = ""):
  88. try:
  89. if task and task.instance:
  90. task.instance.cleanup()
  91. ref_str = f" for task={task.task_ref}" if task.task_ref else ""
  92. self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
  93. except Exception as e:
  94. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  95. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  96. removed = False
  97. with self.m_lock:
  98. if task in self.m_tasks:
  99. self.m_tasks.remove(task)
  100. removed = True
  101. self._on_task_removed(task)
  102. if cleanup and removed:
  103. self._cleanup_task(task, reason)
  104. return removed
  105. def _on_task_removed(self, task: Task):
  106. """子类在 task 被移除时的自定义 Hook(如清理缓存)"""
  107. pass
  108. def _cleanup_all_tasks(self, reason: str = ""):
  109. with self.m_lock:
  110. tasks = list(self.m_tasks)
  111. self.m_tasks.clear()
  112. self._on_all_tasks_cleaned()
  113. for task in tasks:
  114. self._cleanup_task(task, reason)
  115. def _on_all_tasks_cleaned(self):
  116. """子类在所有 task 被清空时的自定义 Hook"""
  117. pass
  118. def _get_redis_key(self, routing_key: str) -> str:
  119. return f"vs:signal:{routing_key}"
  120. def _is_within_active_hours(self) -> bool:
  121. """判断当前是否在允许创建实例的时间段内"""
  122. start_str = self.m_cfg.active_time_start
  123. end_str = self.m_cfg.active_time_end
  124. current_bj_time = datetime.utcnow().time()
  125. start_time = datetime.strptime(start_str, "%H:%M").time()
  126. end_time = datetime.strptime(end_str, "%H:%M").time()
  127. return start_time <= current_bj_time <= end_time
  128. def _maintain_loop(self):
  129. self._log("Maintain loop started.")
  130. while not self.m_stop_event.is_set():
  131. try:
  132. time.sleep(1.0)
  133. with self.m_lock:
  134. tasks_to_check = list(self.m_tasks)
  135. if not tasks_to_check:
  136. continue
  137. dead_tasks = []
  138. healthy_tasks = []
  139. now = time.time()
  140. for t in tasks_to_check:
  141. if now >= t.next_remote_ping:
  142. t.instance.keep_alive()
  143. if t.instance.health_check():
  144. healthy_tasks.append(t)
  145. t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
  146. else:
  147. dead_tasks.append(t)
  148. else:
  149. healthy_tasks.append(t)
  150. self._on_maintain_ping(healthy_tasks, dead_tasks)
  151. if dead_tasks:
  152. with self.m_lock:
  153. current_tasks = list(self.m_tasks)
  154. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  155. for t in dead_tasks:
  156. if t in current_tasks:
  157. self._cleanup_task(t, "unhealthy or keep-alive failed")
  158. else:
  159. with self.m_lock:
  160. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  161. except Exception as e:
  162. self._log(f'Maintain loop exception: {e}')
  163. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  164. """子类维护循环中对于健康/异常 Task 的额外处理"""
  165. pass
  166. def _is_date_of_interest(self, task: Task, query_result: VSQueryResult) -> bool:
  167. """判断 query_result 中的可用日期是否在 task 意向范围内(默认 True,Order 模式重写)"""
  168. return True
  169. def _booking_trigger_loop(self):
  170. self._log("Trigger loop started.")
  171. while not self.m_stop_event.is_set():
  172. try:
  173. time.sleep(0.1)
  174. now = time.time()
  175. for apt_type in self.m_cfg.appointment_types:
  176. redis_key = self._get_redis_key(apt_type.routing_key)
  177. raw_data = self.redis_client.get(redis_key)
  178. if not raw_data:
  179. continue
  180. try:
  181. data = json.loads(raw_data)
  182. query_result = VSQueryResult.model_validate(data['query_result'])
  183. query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
  184. except Exception as parse_err:
  185. self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
  186. self.redis_client.delete(redis_key)
  187. continue
  188. matching_tasks = []
  189. with self.m_lock:
  190. for task in self.m_tasks:
  191. if now < task.next_run or not task.book_allowed:
  192. continue
  193. if apt_type.routing_key not in task.acceptable_routing_keys:
  194. continue
  195. if not self._is_date_of_interest(task, query_result):
  196. continue
  197. task.next_run = now + self.m_cfg.booker.booking_cooldown
  198. matching_tasks.append(task)
  199. if matching_tasks:
  200. threads = []
  201. for task in matching_tasks:
  202. self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
  203. t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
  204. threads.append(t)
  205. t.start()
  206. for t in threads:
  207. t.join()
  208. except Exception as e:
  209. self._log(f"Booking trigger loop exception: {e}")
  210. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  211. raise NotImplementedError
  212. def _creator_loop(self):
  213. raise NotImplementedError
  214. def _push_success_notification(self, task_id, order_id, book_res):
  215. """成功落单后同步云端与推送微信通知的公共方法"""
  216. grab_info = {
  217. "account": book_res.account,
  218. "session_id": book_res.session_id,
  219. "urn": book_res.urn,
  220. "slot_date": book_res.book_date,
  221. "slot_time": book_res.book_time,
  222. "timestamp": int(time.time()),
  223. "payment_link": book_res.payment_link
  224. }
  225. def _update_cloud_success():
  226. try:
  227. VSCloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
  228. push_content = (
  229. f"🎉 【预定成功通知】\n"
  230. f"━━━━━━━━━━━━━━━\n"
  231. f"订单编号: {order_id}\n"
  232. f"预约账号: {book_res.account}\n"
  233. f"预约日期: {book_res.book_date}\n"
  234. f"预约时间: {book_res.book_time}\n"
  235. f"预约编号: {book_res.urn}\n"
  236. f"支付链接: {book_res.payment_link if book_res.payment_link else '无需支付/暂无'}\n"
  237. f"━━━━━━━━━━━━━━━\n"
  238. )
  239. VSCloudApi.Instance().push_weixin_text(push_content)
  240. except Exception as e:
  241. self._log(f"Failed to update success state to cloud: {e}")
  242. ThreadPool.getInstance().enqueue(_update_cloud_success)
  243. self.redis_client.zrem(self.m_tracker_key, task_id)
  244. class BuiltinBookerGCO(BaseBookerGCO):
  245. """
  246. 非绑定模式 (公共内置账号池):
  247. - 只维护全局 target_instances 数量的实例。
  248. - 所有实例热机等待,发现信号后临时去云端 Pop 订单。
  249. """
  250. TAG = "BUILTIN-BOOKER"
  251. def _creator_loop(self):
  252. self._log("Creator loop started.")
  253. while not self.m_stop_event.wait(1.0):
  254. try:
  255. if not self._is_within_active_hours():
  256. continue
  257. if time.time() < self.m_spawn_cooldown_until:
  258. continue
  259. with self.m_lock:
  260. current = len(self.m_tasks)
  261. target = self.m_cfg.booker.target_instances
  262. if current < target:
  263. self._spawn_worker()
  264. except Exception as e:
  265. self._log(f'Creator loop exception: {e}')
  266. def _spawn_worker(self):
  267. instance = None
  268. task = None
  269. success = False
  270. plg_cfg = None
  271. try:
  272. plg_cfg = VSPlgConfig()
  273. plg_cfg.debug = self.m_cfg.debug
  274. plg_cfg.free_config = self.m_cfg.free_config
  275. plg_cfg.session_max_life = self.m_cfg.session_max_life
  276. if self.m_cfg.need_account:
  277. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  278. plg_cfg.account = type(plg_cfg.account)(**acc)
  279. if self.m_cfg.need_proxy:
  280. proxy = VSCloudApi.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  281. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  282. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  283. instance.set_log(self.m_logger)
  284. instance.set_config(plg_cfg)
  285. instance.create_session()
  286. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  287. task = Task(
  288. instance=instance,
  289. next_run=time.time(),
  290. task_ref=None,
  291. acceptable_routing_keys=all_keys,
  292. source_queue="built-in",
  293. book_allowed=True,
  294. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  295. )
  296. with self.m_lock:
  297. self.m_tasks.append(task)
  298. success = True
  299. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  300. except Exception as e:
  301. err_str = str(e)
  302. self.m_spawn_cooldown_until = time.time() + 30.0
  303. self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
  304. rate_limited_indicators = [
  305. "42901" in err_str,
  306. "Rate limited" in err_str
  307. ]
  308. if any(rate_limited_indicators):
  309. if plg_cfg and plg_cfg.account.username != "Guest":
  310. VSCloudApi.lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  311. finally:
  312. if not success:
  313. if task:
  314. self._remove_task(task, "spawn failed")
  315. elif instance:
  316. try:
  317. instance.cleanup()
  318. except Exception:
  319. pass
  320. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  321. queue_name = f"auto.{query_result.apt_type.routing_key}"
  322. task_id = None
  323. task_data = None
  324. try:
  325. task_data = VSCloudApi.Instance().get_vas_task_pop(queue_name)
  326. if not task_data:
  327. return
  328. task_id = task_data['id']
  329. order_id = task_data.get('order_id')
  330. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
  331. user_input = task_data.get('user_inputs', {})
  332. book_res = task.instance.book(query_result, user_input)
  333. if book_res.success:
  334. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  335. self._push_success_notification(task_id, order_id, book_res)
  336. task.successful_bookings += 1
  337. max_b = self.m_cfg.booker.max_bookings_per_account
  338. if max_b > 0 and task.successful_bookings >= max_b:
  339. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  340. self._remove_task(task, "max bookings reached")
  341. else:
  342. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  343. except Exception as e:
  344. err_str = str(e)
  345. self._log(f"Exception during booking: {err_str}")
  346. rate_limited_indicators = [
  347. "42901" in err_str,
  348. "Rate limited" in err_str
  349. ]
  350. if any(rate_limited_indicators):
  351. self._remove_task(task, "booking rate limited")
  352. class OrderBookerGCO(BaseBookerGCO):
  353. """
  354. 绑定模式 (订单自带账号):
  355. - 按城市队列维护热机配额。
  356. - 绝对的 1 对 1 关系:一个实例绑定一个云端订单。
  357. - 预订成功后,实例立即销毁。
  358. """
  359. TAG = "ORDER-BOOKER"
  360. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  361. super().__init__(cfg, redis_conf, logger)
  362. self.m_task_data_cache: Dict[str, dict] = {}
  363. self.heartbeat_ttl = 2 * 60.0
  364. def _start_additional_threads(self):
  365. threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
  366. def _on_task_removed(self, task: Task):
  367. task_id = task.task_ref
  368. if task_id:
  369. self.m_task_data_cache.pop(str(task_id), None)
  370. def _on_all_tasks_cleaned(self):
  371. self.m_task_data_cache.clear()
  372. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  373. if healthy_tasks:
  374. new_deadline = time.time() + self.heartbeat_ttl
  375. mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
  376. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  377. if dead_tasks:
  378. mapping = {str(t.task_ref): 0 for t in dead_tasks}
  379. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  380. def _cache_refresh_loop(self):
  381. self._log("Cache refresh loop started.")
  382. refresh_interval = 15 * 60
  383. while not self.m_stop_event.is_set():
  384. try:
  385. time.sleep(1)
  386. with self.m_lock:
  387. tasks_to_check = {
  388. tid: data.get('_last_refresh', 0)
  389. for tid, data in self.m_task_data_cache.items()
  390. }
  391. if not tasks_to_check:
  392. continue
  393. now = time.time()
  394. for tid, last_refresh in tasks_to_check.items():
  395. if now - last_refresh >= refresh_interval:
  396. fresh_data = VSCloudApi.Instance().get_vas_task(tid)
  397. if fresh_data:
  398. fresh_data['_last_refresh'] = time.time()
  399. with self.m_lock:
  400. if tid in self.m_task_data_cache:
  401. self.m_task_data_cache[tid] = fresh_data
  402. time.sleep(0.5)
  403. except Exception as e:
  404. self._log(f'Cache refresh loop exception: {e}')
  405. def _is_date_of_interest(self, task: Task, query_result: VSQueryResult) -> bool:
  406. if query_result.availability_status != AvailabilityStatus.Available:
  407. return True
  408. task_id = task.task_ref
  409. task_data = self.m_task_data_cache.get(str(task_id), {})
  410. user_input = task_data.get('user_inputs', {})
  411. expected_end_date = (
  412. user_input.get('expected_end_date')
  413. or '2100-01-01'
  414. )
  415. available_date = query_result.earliest_date
  416. dt = available_date.strftime("%Y-%m-%d")
  417. return dt <= expected_end_date
  418. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  419. task_id = task.task_ref
  420. task_data = None
  421. try:
  422. with self.m_lock:
  423. task_data = self.m_task_data_cache.get(str(task_id))
  424. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
  425. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  426. self._remove_task(task, "bound task no longer valid")
  427. self.redis_client.zrem(self.m_tracker_key, task_id)
  428. return
  429. order_id = task_data.get('order_id')
  430. user_input = task_data.get('user_inputs', {})
  431. book_res = task.instance.book(query_result, user_input)
  432. if book_res.success:
  433. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  434. self._push_success_notification(task_id, order_id, book_res)
  435. self._remove_task(task, "booking success")
  436. else:
  437. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  438. except Exception as e:
  439. err_str = str(e)
  440. self._log(f"Exception during booking: {err_str}")
  441. rate_limited_indicators = [
  442. "42901" in err_str,
  443. "Rate limited" in err_str
  444. ]
  445. if any(rate_limited_indicators):
  446. self._remove_task(task, "booking rate limited")
  447. def _creator_loop(self):
  448. self._log("Creator loop started.")
  449. while not self.m_stop_event.wait(1.0):
  450. try:
  451. if not self._is_within_active_hours():
  452. continue
  453. if time.time() < self.m_spawn_cooldown_until:
  454. continue
  455. for apt in self.m_cfg.appointment_types:
  456. r_key = apt.routing_key
  457. with self.m_lock:
  458. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  459. target = self.m_cfg.booker.target_instances
  460. if active < target:
  461. self._spawn_worker(r_key)
  462. except Exception as e:
  463. self._log(f'Creator loop exception:{e}')
  464. def _spawn_worker(self, target_routing_key: str):
  465. instance = None
  466. task = None
  467. success = False
  468. task_id = None
  469. try:
  470. queue_name = f"auto.{target_routing_key}"
  471. task_data = VSCloudApi.Instance().get_vas_task_pop(queue_name)
  472. if not task_data:
  473. return
  474. task_id = task_data['id']
  475. with self.m_lock:
  476. self.m_task_data_cache[str(task_id)] = task_data
  477. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
  478. user_inputs = task_data.get('user_inputs', {})
  479. plg_cfg = VSPlgConfig()
  480. plg_cfg.debug = self.m_cfg.debug
  481. plg_cfg.free_config = self.m_cfg.free_config
  482. plg_cfg.session_max_life = self.m_cfg.session_max_life
  483. plg_cfg.account.username = user_inputs.get("username", "")
  484. plg_cfg.account.password = user_inputs.get("password", "")
  485. if not plg_cfg.account.username:
  486. return
  487. acceptable_keys = [target_routing_key]
  488. if self.m_cfg.need_proxy:
  489. proxy = VSCloudApi.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  490. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  491. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  492. instance.set_log(self.m_logger)
  493. instance.set_config(plg_cfg)
  494. instance.create_session()
  495. task = Task(
  496. instance=instance,
  497. next_run=time.time(),
  498. task_ref=task_id,
  499. acceptable_routing_keys=acceptable_keys,
  500. source_queue=target_routing_key,
  501. book_allowed=True,
  502. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  503. )
  504. with self.m_lock:
  505. self.m_tasks.append(task)
  506. success = True
  507. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  508. except Exception as e:
  509. err_str = str(e)
  510. self.m_spawn_cooldown_until = time.time() + 30.0
  511. self._log(f"Order Booker spawn failed: {err_str}. Backing off for 30s...")
  512. rate_limited_indicators = [
  513. "42901" in err_str,
  514. "Rate limited" in err_str
  515. ]
  516. if any(rate_limited_indicators):
  517. if task_id is not None:
  518. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + self.m_cfg.login_backoff})
  519. finally:
  520. if not success:
  521. if task:
  522. self._remove_task(task, "spawn failed")
  523. elif instance:
  524. try:
  525. instance.cleanup()
  526. except Exception:
  527. pass
  528. if task_id:
  529. with self.m_lock:
  530. self.m_task_data_cache.pop(str(task_id), None)