booker.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. import os
  2. import time
  3. import json
  4. import threading
  5. import random
  6. from datetime import datetime
  7. from typing import List, Dict, Callable, Optional
  8. from vs_types import (
  9. GroupConfig, VSPlgConfig, Task, VSBookResult, VSQueryResult,
  10. AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str
  11. )
  12. from vs_plg_factory import VSPlgFactory
  13. from toolkit.vs_cloud_api import VSCloudApi
  14. from toolkit.proxy_manager import ProxyManager
  15. from toolkit.thread_pool import ThreadPool
  16. from toolkit.notifier import notify_book_result
  17. from utils.safe_redis_cli import SafeRedisClient
  18. class BaseBookerGCO:
  19. """
  20. Booker 基类,封装公共的基础设施与通用的生命周期管理、逻辑循环等。
  21. """
  22. TAG = "BOOKER"
  23. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  24. self.m_cfg = cfg
  25. self.m_factory = VSPlgFactory()
  26. self.m_logger = logger
  27. self.m_tasks: List[Task] = []
  28. self.m_lock = threading.RLock()
  29. self.m_stop_event = threading.Event()
  30. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  31. self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
  32. self.m_last_group_query_time = 0.0
  33. self.m_spawn_cooldown_until = 0.0
  34. def _log(self, message):
  35. prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
  36. if self.m_logger:
  37. self.m_logger(f'{prefix} {message}')
  38. else:
  39. print(f'{prefix} {message}')
  40. def get_status(self) -> GroupStatus:
  41. with self.m_lock:
  42. tasks = list(self.m_tasks)
  43. gid = self.m_cfg.identifier
  44. enable = self.m_cfg.enable
  45. plugin_name = self.m_cfg.plugin_config.plugin_name
  46. plugins_status = []
  47. healthy_count = 0
  48. for t in tasks:
  49. if t and t.instance:
  50. st = t.instance.get_plugin_status()
  51. if st.is_healthy:
  52. healthy_count += 1
  53. plugins_status.append(st)
  54. return GroupStatus(
  55. group_id=gid,
  56. enable=enable,
  57. plugin_name=plugin_name,
  58. total_tasks=len(tasks),
  59. healthy_tasks=healthy_count,
  60. plugins=plugins_status,
  61. updated_at=utc_now_str(),
  62. )
  63. def start(self):
  64. if not self.m_cfg.enable:
  65. return
  66. self._log(f"Starting {self.TAG}...")
  67. plugin_name = self.m_cfg.plugin_config.plugin_name
  68. class_name = "".join(part.title() for part in plugin_name.split('_'))
  69. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  70. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  71. threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
  72. threading.Thread(target=self._creator_loop, daemon=True).start()
  73. threading.Thread(target=self._maintain_loop, daemon=True).start()
  74. self._start_additional_threads()
  75. def _start_additional_threads(self):
  76. """子类扩展线程的 Hook"""
  77. pass
  78. def stop(self):
  79. self._log("Stopping Booker...")
  80. self.m_stop_event.set()
  81. self._cleanup_all_tasks("booker stop")
  82. def update_config(self, new_cfg: GroupConfig):
  83. """动态更新配置"""
  84. with self.m_lock:
  85. if self.m_cfg.enable and not new_cfg.enable:
  86. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  87. elif not self.m_cfg.enable and new_cfg.enable:
  88. self._log("Config dynamically updated: Group ENABLED.")
  89. else:
  90. self._log("Config dynamically updated: Parameters refreshed.")
  91. self.m_cfg = new_cfg
  92. def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
  93. try:
  94. if instance:
  95. ref_str = f" for task={task_ref}" if task_ref else ""
  96. self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
  97. instance.cleanup()
  98. except Exception as e:
  99. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  100. def _cleanup_task(self, task: Task, reason: str = ""):
  101. if task and task.instance:
  102. self._cleanup_instance(task.instance, reason, task.task_ref)
  103. def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
  104. removed = False
  105. with self.m_lock:
  106. if task in self.m_tasks:
  107. self.m_tasks.remove(task)
  108. removed = True
  109. self._on_task_removed(task)
  110. if cleanup and removed:
  111. self._cleanup_task(task, reason)
  112. return removed
  113. def _on_task_removed(self, task: Task):
  114. """子类在 task 被移除时的自定义 Hook(如清理缓存)"""
  115. pass
  116. def _cleanup_all_tasks(self, reason: str = ""):
  117. with self.m_lock:
  118. tasks = list(self.m_tasks)
  119. self.m_tasks.clear()
  120. self._on_all_tasks_cleaned()
  121. for task in tasks:
  122. self._cleanup_task(task, reason)
  123. def _on_all_tasks_cleaned(self):
  124. """子类在所有 task 被清空时的自定义 Hook"""
  125. pass
  126. def _get_redis_key(self, routing_key: str) -> str:
  127. return f"vs:signal:{routing_key}"
  128. def _is_within_active_hours(self) -> bool:
  129. """判断当前是否在允许创建实例的时间段内"""
  130. start_str = self.m_cfg.active_time_start
  131. end_str = self.m_cfg.active_time_end
  132. current_bj_time = datetime.utcnow().time()
  133. start_time = datetime.strptime(start_str, "%H:%M").time()
  134. end_time = datetime.strptime(end_str, "%H:%M").time()
  135. return start_time <= current_bj_time <= end_time
  136. def _maintain_loop(self):
  137. self._log("Maintain loop started.")
  138. while not self.m_stop_event.is_set():
  139. try:
  140. time.sleep(1.0)
  141. with self.m_lock:
  142. tasks_to_check = list(self.m_tasks)
  143. if not tasks_to_check:
  144. continue
  145. dead_tasks = []
  146. healthy_tasks = []
  147. now = time.time()
  148. for t in tasks_to_check:
  149. if now >= t.next_remote_ping:
  150. t.instance.keep_alive()
  151. if t.instance.health_check():
  152. healthy_tasks.append(t)
  153. t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
  154. else:
  155. dead_tasks.append(t)
  156. else:
  157. healthy_tasks.append(t)
  158. self._on_maintain_ping(healthy_tasks, dead_tasks)
  159. if dead_tasks:
  160. with self.m_lock:
  161. current_tasks = list(self.m_tasks)
  162. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  163. for t in dead_tasks:
  164. if t in current_tasks:
  165. self._cleanup_task(t, "unhealthy or keep-alive failed")
  166. else:
  167. with self.m_lock:
  168. self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks]
  169. except Exception as e:
  170. self._log(f'Maintain loop exception: {e}')
  171. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  172. """子类维护循环中对于健康/异常 Task 的额外处理"""
  173. pass
  174. def _is_date_of_interest(self, task: Task, query_result: VSQueryResult) -> bool:
  175. """判断 query_result 中的可用日期是否在 task 意向范围内(默认 True,Order 模式重写)"""
  176. return True
  177. def _booking_trigger_loop(self):
  178. self._log("Trigger loop started.")
  179. while not self.m_stop_event.is_set():
  180. try:
  181. time.sleep(0.1)
  182. now = time.time()
  183. for apt_type in self.m_cfg.appointment_types:
  184. redis_key = self._get_redis_key(apt_type.routing_key)
  185. raw_data = self.redis_client.get(redis_key)
  186. if not raw_data:
  187. continue
  188. try:
  189. data = json.loads(raw_data)
  190. query_result = VSQueryResult.model_validate(data['query_result'])
  191. query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
  192. except Exception as parse_err:
  193. self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
  194. self.redis_client.delete(redis_key)
  195. continue
  196. matching_tasks = []
  197. with self.m_lock:
  198. for task in self.m_tasks:
  199. if now < task.next_run or not task.book_allowed:
  200. continue
  201. if apt_type.routing_key not in task.acceptable_routing_keys:
  202. continue
  203. if not self._is_date_of_interest(task, query_result):
  204. continue
  205. task.next_run = now + self.m_cfg.booker.booking_cooldown
  206. matching_tasks.append(task)
  207. if matching_tasks:
  208. threads = []
  209. for task in matching_tasks:
  210. self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
  211. t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
  212. threads.append(t)
  213. t.start()
  214. for t in threads:
  215. t.join()
  216. except Exception as e:
  217. self._log(f"Booking trigger loop exception: {e}")
  218. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  219. raise NotImplementedError
  220. def _creator_loop(self):
  221. raise NotImplementedError
  222. def _push_success_notification(self, task_id, order_id, book_res):
  223. """成功落单后同步云端与推送微信通知的公共方法"""
  224. grab_info = {
  225. "account": book_res.account,
  226. "session_id": book_res.session_id,
  227. "urn": book_res.urn,
  228. "slot_date": book_res.book_date,
  229. "slot_time": book_res.book_time,
  230. "timestamp": int(time.time()),
  231. "payment_link": book_res.payment_link
  232. }
  233. def _update_cloud_success():
  234. try:
  235. VSCloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
  236. except Exception as e:
  237. self._log(f"Failed to update success state to cloud: {e}")
  238. ThreadPool.getInstance().enqueue(_update_cloud_success)
  239. notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
  240. self.redis_client.zrem(self.m_tracker_key, task_id)
  241. class BuiltinBookerGCO(BaseBookerGCO):
  242. """
  243. 非绑定模式 (公共内置账号池):
  244. - 只维护全局 target_instances 数量的实例。
  245. - 所有实例热机等待,发现信号后临时去云端 Pop 订单。
  246. """
  247. TAG = "BUILTIN-BOOKER"
  248. def _creator_loop(self):
  249. self._log("Creator loop started.")
  250. while not self.m_stop_event.wait(1.0):
  251. if not self._is_within_active_hours():
  252. continue
  253. if time.time() < self.m_spawn_cooldown_until:
  254. continue
  255. with self.m_lock:
  256. current = len(self.m_tasks)
  257. target = self.m_cfg.booker.target_instances
  258. if current < target:
  259. self._spawn_worker()
  260. def _spawn_worker(self):
  261. instance = None
  262. task = None
  263. success = False
  264. plg_cfg = None
  265. try:
  266. plg_cfg = VSPlgConfig()
  267. plg_cfg.debug = self.m_cfg.debug
  268. plg_cfg.free_config = self.m_cfg.free_config
  269. plg_cfg.session_max_life = self.m_cfg.session_max_life
  270. if self.m_cfg.need_account:
  271. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
  272. plg_cfg.account = type(plg_cfg.account)(**acc)
  273. if self.m_cfg.need_proxy:
  274. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  275. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  276. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  277. instance.set_log(self.m_logger)
  278. instance.set_config(plg_cfg)
  279. instance.create_session()
  280. all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
  281. task = Task(
  282. instance=instance,
  283. next_run=time.time(),
  284. task_ref=None,
  285. acceptable_routing_keys=all_keys,
  286. source_queue="built-in",
  287. book_allowed=True,
  288. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  289. )
  290. with self.m_lock:
  291. self.m_tasks.append(task)
  292. success = True
  293. self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
  294. except Exception as e:
  295. err_str = str(e)
  296. self.m_spawn_cooldown_until = time.time() + 30.0
  297. self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
  298. rate_limited_indicators = [
  299. "42901" in err_str,
  300. "Rate limited" in err_str
  301. ]
  302. if any(rate_limited_indicators):
  303. if plg_cfg and plg_cfg.account.username != "Guest":
  304. VSCloudApi.Instance().lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  305. finally:
  306. if not success and instance:
  307. self._cleanup_instance(instance, "spawn failed")
  308. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  309. queue_name = f"auto.{query_result.apt_type.routing_key}"
  310. task_id = None
  311. task_data = None
  312. try:
  313. task_data = VSCloudApi.Instance().get_vas_task_pop(queue_name)
  314. if not task_data:
  315. return
  316. task_id = task_data['id']
  317. order_id = task_data.get('order_id')
  318. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
  319. user_input = task_data.get('user_inputs', {})
  320. book_res = task.instance.book(query_result, user_input)
  321. if book_res.success:
  322. self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
  323. self._push_success_notification(task_id, order_id, book_res)
  324. task.successful_bookings += 1
  325. max_b = self.m_cfg.booker.max_bookings_per_account
  326. if max_b > 0 and task.successful_bookings >= max_b:
  327. self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
  328. self._remove_task(task, "max bookings reached")
  329. else:
  330. self._log(f"❌ BOOK FAILED for Order: {order_id}")
  331. except Exception as e:
  332. err_str = str(e)
  333. self._log(f"Exception during booking: {err_str}")
  334. rate_limited_indicators = [
  335. "42901" in err_str,
  336. "Rate limited" in err_str
  337. ]
  338. if any(rate_limited_indicators):
  339. self._remove_task(task, "booking rate limited")
  340. class OrderBookerGCO(BaseBookerGCO):
  341. """
  342. 绑定模式 (订单自带账号):
  343. - 按城市队列维护热机配额。
  344. - 绝对的 1 对 1 关系:一个实例绑定一个云端订单。
  345. - 预订成功后,实例立即销毁。
  346. """
  347. TAG = "ORDER-BOOKER"
  348. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  349. super().__init__(cfg, redis_conf, logger)
  350. self.m_task_data_cache: Dict[str, dict] = {}
  351. self.heartbeat_ttl = 2 * 60.0
  352. def _start_additional_threads(self):
  353. threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
  354. def _on_task_removed(self, task: Task):
  355. task_id = task.task_ref
  356. if task_id:
  357. self.m_task_data_cache.pop(str(task_id), None)
  358. def _on_all_tasks_cleaned(self):
  359. self.m_task_data_cache.clear()
  360. def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
  361. if healthy_tasks:
  362. new_deadline = time.time() + self.heartbeat_ttl
  363. mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
  364. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  365. if dead_tasks:
  366. mapping = {str(t.task_ref): 0 for t in dead_tasks}
  367. self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
  368. def _cache_refresh_loop(self):
  369. self._log("Cache refresh loop started.")
  370. refresh_interval = 15 * 60
  371. while not self.m_stop_event.is_set():
  372. try:
  373. time.sleep(1)
  374. with self.m_lock:
  375. tasks_to_check = {
  376. tid: data.get('_last_refresh', 0)
  377. for tid, data in self.m_task_data_cache.items()
  378. }
  379. if not tasks_to_check:
  380. continue
  381. now = time.time()
  382. for tid, last_refresh in tasks_to_check.items():
  383. if now - last_refresh >= refresh_interval:
  384. fresh_data = VSCloudApi.Instance().get_vas_task(tid)
  385. if fresh_data:
  386. fresh_data['_last_refresh'] = time.time()
  387. with self.m_lock:
  388. if tid in self.m_task_data_cache:
  389. self.m_task_data_cache[tid] = fresh_data
  390. time.sleep(0.5)
  391. except Exception as e:
  392. self._log(f'Cache refresh loop exception: {e}')
  393. def _is_date_of_interest(self, task: Task, query_result: VSQueryResult) -> bool:
  394. if query_result.availability_status != AvailabilityStatus.Available:
  395. return True
  396. task_id = task.task_ref
  397. task_data = self.m_task_data_cache.get(str(task_id), {})
  398. user_input = task_data.get('user_inputs', {})
  399. expected_end_date = (
  400. user_input.get('expected_end_date')
  401. or '2100-01-01'
  402. )
  403. available_date = query_result.earliest_date
  404. dt = available_date.strftime("%Y-%m-%d")
  405. return dt <= expected_end_date
  406. def _execute_book_job(self, task: Task, query_result: VSQueryResult):
  407. task_id = task.task_ref
  408. task_data = None
  409. try:
  410. with self.m_lock:
  411. task_data = self.m_task_data_cache.get(str(task_id))
  412. if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
  413. self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
  414. self._remove_task(task, "bound task no longer valid")
  415. self.redis_client.zrem(self.m_tracker_key, task_id)
  416. return
  417. order_id = task_data.get('order_id')
  418. user_input = task_data.get('user_inputs', {})
  419. book_res = task.instance.book(query_result, user_input)
  420. if book_res.success:
  421. self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
  422. self._push_success_notification(task_id, order_id, book_res)
  423. self._remove_task(task, "booking success")
  424. else:
  425. self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
  426. except Exception as e:
  427. err_str = str(e)
  428. self._log(f"Exception during booking: {err_str}")
  429. rate_limited_indicators = [
  430. "42901" in err_str,
  431. "Rate limited" in err_str
  432. ]
  433. if any(rate_limited_indicators):
  434. self._remove_task(task, "booking rate limited")
  435. def _creator_loop(self):
  436. self._log("Creator loop started.")
  437. while not self.m_stop_event.wait(1.0):
  438. if not self._is_within_active_hours():
  439. continue
  440. if time.time() < self.m_spawn_cooldown_until:
  441. continue
  442. for apt in self.m_cfg.appointment_types:
  443. r_key = apt.routing_key
  444. with self.m_lock:
  445. active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
  446. target = self.m_cfg.booker.target_instances
  447. if active < target:
  448. self._spawn_worker(r_key)
  449. def _spawn_worker(self, target_routing_key: str):
  450. instance = None
  451. task = None
  452. success = False
  453. task_id = None
  454. try:
  455. queue_name = f"auto.{target_routing_key}"
  456. task_data = VSCloudApi.Instance().get_vas_task_pop(queue_name)
  457. if not task_data:
  458. return
  459. task_id = task_data['id']
  460. with self.m_lock:
  461. self.m_task_data_cache[str(task_id)] = task_data
  462. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
  463. user_inputs = task_data.get('user_inputs', {})
  464. plg_cfg = VSPlgConfig()
  465. plg_cfg.debug = self.m_cfg.debug
  466. plg_cfg.free_config = self.m_cfg.free_config
  467. plg_cfg.session_max_life = self.m_cfg.session_max_life
  468. plg_cfg.account.username = user_inputs.get("username", "")
  469. plg_cfg.account.password = user_inputs.get("password", "")
  470. if not plg_cfg.account.username:
  471. return
  472. acceptable_keys = [target_routing_key]
  473. if self.m_cfg.need_proxy:
  474. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  475. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  476. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  477. instance.set_log(self.m_logger)
  478. instance.set_config(plg_cfg)
  479. instance.create_session()
  480. task = Task(
  481. instance=instance,
  482. next_run=time.time(),
  483. task_ref=task_id,
  484. acceptable_routing_keys=acceptable_keys,
  485. source_queue=target_routing_key,
  486. book_allowed=True,
  487. next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
  488. )
  489. with self.m_lock:
  490. self.m_tasks.append(task)
  491. success = True
  492. self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
  493. except Exception as e:
  494. err_str = str(e)
  495. self.m_spawn_cooldown_until = time.time() + 30.0
  496. self._log(f"Order Booker spawn failed: {err_str}. Backing off for 30s...")
  497. rate_limited_indicators = [
  498. "42901" in err_str,
  499. "Rate limited" in err_str
  500. ]
  501. if any(rate_limited_indicators):
  502. if task_id is not None:
  503. self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + self.m_cfg.login_backoff})
  504. finally:
  505. if not success and instance:
  506. self._cleanup_instance(instance, "spawn failed")
  507. if not success and task_id:
  508. with self.m_lock:
  509. self.m_task_data_cache.pop(str(task_id), None)