booker.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604
  1. import os
  2. import time
  3. import json
  4. import threading
  5. import random
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from vs_types import GroupConfig, VSPlgConfig, Task, VSQueryResult, AppointmentType, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str
  9. from vs_plg_factory import VSPlgFactory
  10. from toolkit.thread_pool import ThreadPool
  11. from toolkit.vs_cloud_api import VSCloudApi
  12. from utils.safe_redis_cli import SafeRedisClient
  13. class BaseBookerGCO:
  14. """
  15. Booker 基类,封装公共的基础设施与通用的生命周期管理、逻辑循环等。
  16. """
  17. TAG = "BOOKER"
  18. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  19. self.m_cfg = cfg
  20. self.m_factory = VSPlgFactory()
  21. self.m_logger = logger
  22. self.m_tasks: List[Task] = []
  23. self.m_lock = threading.RLock()
  24. self.m_stop_event = threading.Event()
  25. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  26. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  27. self.m_last_group_query_time = 0.0
  28. def _log(self, message):
  29. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  30. if self.m_logger:
  31. self.m_logger(f'{prefix} {message}')
  32. else:
  33. print(f'{prefix} {message}')
  34. def get_status(self) -> GroupStatus:
  35. with self.m_lock:
  36. tasks = list(self.m_tasks)
  37. gid = self.m_cfg.identifier
  38. enable = self.m_cfg.enable
  39. plugin_name = self.m_cfg.plugin_config.plugin_name
  40. plugins_status = []
  41. healthy_count = 0
  42. for t in tasks:
  43. if t and t.instance:
  44. st = t.instance.get_plugin_status()
  45. if st.is_healthy:
  46. healthy_count += 1
  47. plugins_status.append(st)
  48. return GroupStatus(
  49. group_id=gid,
  50. enable=enable,
  51. plugin_name=plugin_name,
  52. total_tasks=len(tasks),
  53. healthy_tasks=healthy_count,
  54. plugins=plugins_status,
  55. updated_at=utc_now_str(),
  56. )
  57. def start(self):
  58. if not self.m_cfg.enable:
  59. return
  60. self._log(f"Starting {self.TAG}...")
  61. plugin_name = self.m_cfg.plugin_config.plugin_name
  62. class_name = "".join(part.title() for part in plugin_name.split('_'))
  63. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  64. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  65. threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
  66. threading.Thread(target=self._creator_loop, daemon=True).start()
  67. threading.Thread(target=self._maintain_loop, daemon=True).start()
  68. self._start_additional_threads()
  69. def _start_additional_threads(self):
  70. """子类扩展线程的 Hook"""
  71. pass
  72. def stop(self):
  73. self._log("Stopping Booker...")
  74. self.m_stop_event.set()
  75. self._cleanup_all_tasks("booker stop")
  76. def update_config(self, new_cfg: GroupConfig):
  77. """动态更新配置"""
  78. with self.m_lock:
  79. if self.m_cfg.enable and not new_cfg.enable:
  80. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  81. elif not self.m_cfg.enable and new_cfg.enable:
  82. self._log("Config dynamically updated: Group ENABLED.")
  83. else:
  84. self._log("Config dynamically updated: Parameters refreshed.")
  85. self.m_cfg = new_cfg
  86. def _cleanup_task(self, task: Task, reason: str = ""):
  87. try:
  88. if task and task.instance:
  89. task.instance.cleanup()
  90. ref_str = f" for task={task.task_ref}" if task.task_ref else ""
  91. self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
  92. except Exception as e:
  93. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  94. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  95. removed = False
  96. with self.m_lock:
  97. if task in self.m_tasks:
  98. self.m_tasks.remove(task)
  99. removed = True
  100. self._on_task_removed(task)
  101. if cleanup and removed:
  102. self._cleanup_task(task, reason)
  103. return removed
  104. def _on_task_removed(self, task: Task):
  105. """子类在 task 被移除时的自定义 Hook(如清理缓存)"""
  106. pass
  107. def _cleanup_all_tasks(self, reason: str = ""):
  108. with self.m_lock:
  109. tasks = list(self.m_tasks)
  110. self.m_tasks.clear()
  111. self._on_all_tasks_cleaned()
  112. for task in tasks:
  113. self._cleanup_task(task, reason)
  114. def _on_all_tasks_cleaned(self):
  115. """子类在所有 task 被清空时的自定义 Hook"""
  116. pass
  117. def _get_redis_key(self, routing_key: str) -> str:
  118. return f"vs:signal:{routing_key}"
  119. def _is_within_active_hours(self) -> bool:
  120. """判断当前是否在允许创建实例的时间段内"""
  121. start_str = self.m_cfg.active_time_start
  122. end_str = self.m_cfg.active_time_end
  123. current_bj_time = datetime.utcnow().time()
  124. start_time = datetime.strptime(start_str, "%H:%M").time()
  125. end_time = datetime.strptime(end_str, "%H:%M").time()
  126. return start_time <= current_bj_time <= end_time
  127. def _maintain_loop(self):
  128. self._log("Maintain loop started.")
  129. while not self.m_stop_event.is_set():
  130. try:
  131. time.sleep(1.0)
  132. with self.m_lock:
  133. tasks_to_check = list(self.m_tasks)
  134. if not tasks_to_check:
  135. continue
  136. dead_tasks = []
  137. healthy_tasks = []
  138. now = time.time()
  139. for t in tasks_to_check:
  140. if now >= t.next_remote_ping:
  141. t.instance.keep_alive()
  142. if t.instance.health_check():
  143. healthy_tasks.append(t)
  144. t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
  145. else:
  146. dead_tasks.append(t)
  147. else:
  148. healthy_tasks.append(t)
  149. self._on_maintain_ping(healthy_tasks, dead_tasks)
  150. if dead_tasks:
  151. with self.m_lock:
  152. current_tasks = list(self.m_tasks)
  153. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  154. for t in dead_tasks:
  155. if t in current_tasks:
  156. self._cleanup_task(t, "unhealthy or keep-alive failed")
  157. else:
  158. with self.m_lock:
  159. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  160. except Exception as e:
  161. self._log(f'Maintain loop exception: {e}')
  162. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  163. """子类维护循环中对于健康/异常 Task 的额外处理"""
  164. pass
  165. def _is_date_of_interest(self, task: Task, query_result: VSQueryResult) -> bool:
  166. """判断 query_result 中的可用日期是否在 task 意向范围内(默认 True,Order 模式重写)"""
  167. return True
  168. def _booking_trigger_loop(self):
  169. self._log("Trigger loop started.")
  170. while not self.m_stop_event.is_set():
  171. try:
  172. time.sleep(0.1)
  173. now = time.time()
  174. for apt_type in self.m_cfg.appointment_types:
  175. redis_key = self._get_redis_key(apt_type.routing_key)
  176. raw_data = self.redis_client.get(redis_key)
  177. if not raw_data:
  178. continue
  179. try:
  180. data = json.loads(raw_data)
  181. query_result = VSQueryResult.model_validate(data['query_result'])
  182. query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
  183. except Exception as parse_err:
  184. self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
  185. self.redis_client.delete(redis_key)
  186. continue
  187. matching_tasks = []
  188. with self.m_lock:
  189. for task in self.m_tasks:
  190. if now < task.next_run or not task.book_allowed:
  191. continue
  192. if apt_type.routing_key not in task.acceptable_routing_keys:
  193. continue
  194. if not self._is_date_of_interest(task, query_result):
  195. continue
  196. task.next_run = now + self.m_cfg.booker.booking_cooldown
  197. matching_tasks.append(task)
  198. if matching_tasks:
  199. threads = []
  200. for task in matching_tasks:
  201. self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
  202. t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
  203. threads.append(t)
  204. t.start()
  205. for t in threads:
  206. t.join()
  207. except Exception as e:
  208. self._log(f"Booking trigger loop exception: {e}")
  209. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  210. raise NotImplementedError
  211. def _creator_loop(self):
  212. raise NotImplementedError
  213. def _push_success_notification(self, task_id, order_id, book_res):
  214. """成功落单后同步云端与推送微信通知的公共方法"""
  215. grab_info = {
  216. "account": book_res.account,
  217. "session_id": book_res.session_id,
  218. "urn": book_res.urn,
  219. "slot_date": book_res.book_date,
  220. "slot_time": book_res.book_time,
  221. "timestamp": int(time.time()),
  222. "payment_link": book_res.payment_link
  223. }
  224. def _update_cloud_success():
  225. try:
  226. VSCloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
  227. push_content = (
  228. f"🎉 【预定成功通知】\n"
  229. f"━━━━━━━━━━━━━━━\n"
  230. f"订单编号: {order_id}\n"
  231. f"预约账号: {book_res.account}\n"
  232. f"预约日期: {book_res.book_date}\n"
  233. f"预约时间: {book_res.book_time}\n"
  234. f"预约编号: {book_res.urn}\n"
  235. f"支付链接: {book_res.payment_link if book_res.payment_link else '无需支付/暂无'}\n"
  236. f"━━━━━━━━━━━━━━━\n"
  237. )
  238. VSCloudApi.Instance().push_weixin_text(push_content)
  239. except Exception as e:
  240. self._log(f"Failed to update success state to cloud: {e}")
  241. ThreadPool.getInstance().enqueue(_update_cloud_success)
  242. self.redis_client.zrem(self.m_tracker_key, task_id)
  243. class BuiltinBookerGCO(BaseBookerGCO):
  244. """
  245. 非绑定模式 (公共内置账号池):
  246. - 只维护全局 target_instances 数量的实例。
  247. - 所有实例热机等待,发现信号后临时去云端 Pop 订单。
  248. """
  249. TAG = "BUILTIN-BOOKER"
  250. def _creator_loop(self):
  251. self._log("Creator loop started.")
  252. while not self.m_stop_event.wait(1.0):
  253. try:
  254. if not self._is_within_active_hours():
  255. continue
  256. with self.m_lock:
  257. current = len(self.m_tasks)
  258. target = self.m_cfg.booker.target_instances
  259. if current < target:
  260. self._spawn_worker()
  261. except Exception as e:
  262. self._log(f'Creator loop exception: {e}')
  263. def _spawn_worker(self):
  264. instance = None
  265. task = None
  266. success = False
  267. plg_cfg = None
  268. try:
  269. plg_cfg = VSPlgConfig()
  270. plg_cfg.debug = self.m_cfg.debug
  271. plg_cfg.free_config = self.m_cfg.free_config
  272. plg_cfg.session_max_life = self.m_cfg.session_max_life
  273. if self.m_cfg.need_account:
  274. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  275. plg_cfg.account = type(plg_cfg.account)(**acc)
  276. if self.m_cfg.need_proxy:
  277. proxy = VSCloudApi.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  278. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  279. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  280. instance.set_log(self.m_logger)
  281. instance.set_config(plg_cfg)
  282. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  283. task = Task(
  284. instance=instance,
  285. next_run=time.time() + 999999,
  286. task_ref=None,
  287. acceptable_routing_keys=all_keys,
  288. source_queue="built-in",
  289. book_allowed=True,
  290. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  291. )
  292. with self.m_lock:
  293. self.m_tasks.append(task)
  294. instance.create_session()
  295. task.next_run = time.time()
  296. success = True
  297. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  298. except Exception as e:
  299. err_str = str(e)
  300. self._log(f"Spawn failed: {err_str}")
  301. rate_limited_indicators = [
  302. "42901" in err_str,
  303. "Rate limited" in err_str
  304. ]
  305. if any(rate_limited_indicators):
  306. if plg_cfg and plg_cfg.account.username != "Guest":
  307. VSCloudApi.lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  308. finally:
  309. if not success:
  310. if task:
  311. self._remove_task(task, "spawn failed")
  312. elif instance:
  313. instance.cleanup()
  314. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  315. queue_name = f"auto.{query_result.apt_type.routing_key}"
  316. task_id = None
  317. task_data = None
  318. try:
  319. task_data = VSCloudApi.Instance().get_vas_task_pop(queue_name)
  320. if not task_data:
  321. return
  322. task_id = task_data['id']
  323. order_id = task_data.get('order_id')
  324. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
  325. user_input = task_data.get('user_inputs', {})
  326. book_res = task.instance.book(query_result, user_input)
  327. if book_res.success:
  328. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  329. self._push_success_notification(task_id, order_id, book_res)
  330. task.successful_bookings += 1
  331. max_b = self.m_cfg.booker.max_bookings_per_account
  332. if max_b > 0 and task.successful_bookings >= max_b:
  333. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  334. self._remove_task(task, "max bookings reached")
  335. else:
  336. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  337. except Exception as e:
  338. err_str = str(e)
  339. self._log(f"Exception during booking: {err_str}")
  340. rate_limited_indicators = [
  341. "42901" in err_str,
  342. "Rate limited" in err_str
  343. ]
  344. if any(rate_limited_indicators):
  345. self._remove_task(task, "booking rate limited")
  346. class OrderBookerGCO(BaseBookerGCO):
  347. """
  348. 绑定模式 (订单自带账号):
  349. - 按城市队列维护热机配额。
  350. - 绝对的 1 对 1 关系:一个实例绑定一个云端订单。
  351. - 预订成功后,实例立即销毁。
  352. """
  353. TAG = "ORDER-BOOKER"
  354. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  355. super().__init__(cfg, redis_conf, logger)
  356. self.m_task_data_cache: Dict[str, dict] = {}
  357. self.heartbeat_ttl = 2 * 60.0
  358. def _start_additional_threads(self):
  359. threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
  360. def _on_task_removed(self, task: Task):
  361. task_id = task.task_ref
  362. if task_id:
  363. self.m_task_data_cache.pop(str(task_id), None)
  364. def _on_all_tasks_cleaned(self):
  365. self.m_task_data_cache.clear()
  366. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  367. if healthy_tasks:
  368. new_deadline = time.time() + self.heartbeat_ttl
  369. mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
  370. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  371. if dead_tasks:
  372. mapping = {str(t.task_ref): 0 for t in dead_tasks}
  373. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  374. def _cache_refresh_loop(self):
  375. self._log("Cache refresh loop started.")
  376. refresh_interval = 15 * 60
  377. while not self.m_stop_event.is_set():
  378. try:
  379. time.sleep(1)
  380. with self.m_lock:
  381. tasks_to_check = {
  382. tid: data.get('_last_refresh', 0)
  383. for tid, data in self.m_task_data_cache.items()
  384. }
  385. if not tasks_to_check:
  386. continue
  387. now = time.time()
  388. for tid, last_refresh in tasks_to_check.items():
  389. if now - last_refresh >= refresh_interval:
  390. fresh_data = VSCloudApi.Instance().get_vas_task(tid)
  391. if fresh_data:
  392. fresh_data['_last_refresh'] = time.time()
  393. with self.m_lock:
  394. if tid in self.m_task_data_cache:
  395. self.m_task_data_cache[tid] = fresh_data
  396. time.sleep(0.5)
  397. except Exception as e:
  398. self._log(f'Cache refresh loop exception: {e}')
  399. def _is_date_of_interest(self, task: Task, query_result: VSQueryResult) -> bool:
  400. if query_result.availability_status != AvailabilityStatus.Available:
  401. return True
  402. task_id = task.task_ref
  403. task_data = self.m_task_data_cache.get(str(task_id), {})
  404. user_input = task_data.get('user_inputs', {})
  405. expected_end_date = (
  406. user_input.get('expected_end_date')
  407. or '2100-01-01'
  408. )
  409. available_date = query_result.earliest_date
  410. dt = available_date.strftime("%Y-%m-%d")
  411. return dt <= expected_end_date
  412. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  413. task_id = task.task_ref
  414. task_data = None
  415. try:
  416. with self.m_lock:
  417. task_data = self.m_task_data_cache.get(str(task_id))
  418. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
  419. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  420. self._remove_task(task, "bound task no longer valid")
  421. self.redis_client.zrem(self.m_tracker_key, task_id)
  422. return
  423. order_id = task_data.get('order_id')
  424. user_input = task_data.get('user_inputs', {})
  425. book_res = task.instance.book(query_result, user_input)
  426. if book_res.success:
  427. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  428. self._push_success_notification(task_id, order_id, book_res)
  429. self._remove_task(task, "booking success")
  430. else:
  431. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  432. except Exception as e:
  433. err_str = str(e)
  434. self._log(f"Exception during booking: {err_str}")
  435. rate_limited_indicators = [
  436. "42901" in err_str,
  437. "Rate limited" in err_str
  438. ]
  439. if any(rate_limited_indicators):
  440. self._remove_task(task, "booking rate limited")
  441. def _creator_loop(self):
  442. self._log("Creator loop started.")
  443. while not self.m_stop_event.wait(1.0):
  444. try:
  445. if not self._is_within_active_hours():
  446. continue
  447. for apt in self.m_cfg.appointment_types:
  448. r_key = apt.routing_key
  449. with self.m_lock:
  450. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  451. target = self.m_cfg.booker.target_instances
  452. if active < target:
  453. self._spawn_worker(r_key)
  454. except Exception as e:
  455. self._log(f'Creator loop exception:{e}')
  456. def _spawn_worker(self, target_routing_key: str):
  457. instance = None
  458. task = None
  459. success = False
  460. task_id = None
  461. try:
  462. queue_name = f"auto.{target_routing_key}"
  463. task_data = VSCloudApi.Instance().get_vas_task_pop(queue_name)
  464. if not task_data:
  465. return
  466. task_id = task_data['id']
  467. with self.m_lock:
  468. self.m_task_data_cache[str(task_id)] = task_data
  469. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
  470. user_inputs = task_data.get('user_inputs', {})
  471. plg_cfg = VSPlgConfig()
  472. plg_cfg.debug = self.m_cfg.debug
  473. plg_cfg.free_config = self.m_cfg.free_config
  474. plg_cfg.session_max_life = self.m_cfg.session_max_life
  475. plg_cfg.account.username = user_inputs.get("username", "")
  476. plg_cfg.account.password = user_inputs.get("password", "")
  477. if not plg_cfg.account.username:
  478. return
  479. acceptable_keys = [target_routing_key]
  480. if self.m_cfg.need_proxy:
  481. proxy = VSCloudApi.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  482. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  483. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  484. instance.set_log(self.m_logger)
  485. instance.set_config(plg_cfg)
  486. task = Task(
  487. instance=instance,
  488. next_run=time.time() + 999999,
  489. task_ref=task_id,
  490. acceptable_routing_keys=acceptable_keys,
  491. source_queue=target_routing_key,
  492. book_allowed=True,
  493. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  494. )
  495. with self.m_lock:
  496. self.m_tasks.append(task)
  497. instance.create_session()
  498. task.next_run = time.time()
  499. success = True
  500. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  501. except Exception as e:
  502. err_str = str(e)
  503. self._log(f"Order Booker spawn failed: {err_str}")
  504. rate_limited_indicators = [
  505. "42901" in err_str,
  506. "Rate limited" in err_str
  507. ]
  508. if any(rate_limited_indicators):
  509. if task_id is not None:
  510. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + self.m_cfg.login_backoff})
  511. finally:
  512. if not success:
  513. if task:
  514. self._remove_task(task, "spawn failed")
  515. elif instance:
  516. instance.cleanup()
  517. if task_id:
  518. with self.m_lock:
  519. self.m_task_data_cache.pop(str(task_id), None)