import os import time import json import threading import random from datetime import datetime from typing import List, Dict, Callable, Optional from core_types import ( GroupConfig, PluginConfig, Task, BookResult, QueryResult, AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str, BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError, NetworkTimeoutError ) from plugin_factory import PluginFactory from toolkit.cloud_api import CloudApi from toolkit.proxy_manager import ProxyManager from toolkit.thread_pool import ThreadPool from toolkit.notifier import notify_book_result from utils.safe_redis_cli import SafeRedisClient class BaseBookerGCO: """ Booker 基类,封装公共的基础设施与通用的生命周期管理、逻辑循环等。 """ TAG = "BOOKER" def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): self.m_cfg = cfg self.m_factory = PluginFactory() self.m_logger = logger self.m_tasks: List[Task] = [] self.m_lock = threading.RLock() self.m_stop_event = threading.Event() self.redis_client = SafeRedisClient(redis_conf, self.m_logger) self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}" self.m_last_group_query_time = 0.0 self.m_spawn_cooldown_until = 0.0 def _log(self, message): prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]" if self.m_logger: self.m_logger(f'{prefix} {message}') else: print(f'{prefix} {message}') def get_status(self) -> GroupStatus: with self.m_lock: tasks = list(self.m_tasks) gid = self.m_cfg.identifier enable = self.m_cfg.enable plugin_name = self.m_cfg.plugin_config.plugin_name plugins_status = [] healthy_count = 0 for t in tasks: if t and t.instance: st = t.instance.get_plugin_status() if st.is_healthy: healthy_count += 1 plugins_status.append(st) return GroupStatus( group_id=gid, enable=enable, plugin_name=plugin_name, total_tasks=len(tasks), healthy_tasks=healthy_count, plugins=plugins_status, updated_at=utc_now_str(), ) def start(self): if not self.m_cfg.enable: return self._log(f"Starting {self.TAG}...") plugin_name = self.m_cfg.plugin_config.plugin_name class_name = "".join(part.title() for part in plugin_name.split('_')) plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin) self.m_factory.register_plugin(plugin_name, plugin_path, class_name) threading.Thread(target=self._booking_trigger_loop, daemon=True).start() threading.Thread(target=self._creator_loop, daemon=True).start() threading.Thread(target=self._maintain_loop, daemon=True).start() self._start_additional_threads() def _start_additional_threads(self): """子类扩展线程的 Hook""" pass def stop(self): self._log("Stopping Booker...") self.m_stop_event.set() self._cleanup_all_tasks("booker stop") def update_config(self, new_cfg: GroupConfig): """动态更新配置""" with self.m_lock: if self.m_cfg.enable and not new_cfg.enable: self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.") elif not self.m_cfg.enable and new_cfg.enable: self._log("Config dynamically updated: Group ENABLED.") else: self._log("Config dynamically updated: Parameters refreshed.") self.m_cfg = new_cfg def _cleanup_instance(self, instance, reason: str = "", task_ref=None): try: if instance: ref_str = f" for task={task_ref}" if task_ref else "" self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}") instance.cleanup() except Exception as e: self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}") def _cleanup_task(self, task: Task, reason: str = ""): if task and task.instance: self._cleanup_instance(task.instance, reason, task.task_ref) def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True): removed = False with self.m_lock: if task in self.m_tasks: self.m_tasks.remove(task) removed = True self._on_task_removed(task) if cleanup and removed: self._cleanup_task(task, reason) return removed def _on_task_removed(self, task: Task): """子类在 task 被移除时的自定义 Hook(如清理缓存)""" pass def _cleanup_all_tasks(self, reason: str = ""): with self.m_lock: tasks = list(self.m_tasks) self.m_tasks.clear() self._on_all_tasks_cleaned() for task in tasks: self._cleanup_task(task, reason) def _on_all_tasks_cleaned(self): """子类在所有 task 被清空时的自定义 Hook""" pass def _get_redis_key(self, routing_key: str) -> str: return f"vs:signal:{routing_key}" def _normalize_error(self, err: Exception) -> BizException: """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容""" if isinstance(err, BizException): return err err_str = str(err) if "42901" in err_str or "Rate limited" in err_str: return RateLimiteddError(message=err_str) if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower(): return SessionExpiredOrInvalidError(message=err_str) if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str: return PermissionDeniedError(message=err_str) return BizException( code=50000, message=f"Unhandled ({type(err).__name__}): {err_str}", ) def _dispatch_error_policy( self, action: str, err: Exception, task: Optional[Task] = None, plg_cfg: Optional[PluginConfig] = None, task_id: Optional[int] = None ) -> None: """ Booker 错误调度与自愈策略引擎 (Policy Engine): 基于 (Action, 异常事实类型) 矩阵 + Booker 模式闭环执行自愈动作: - spawn/login 时 429:判定为账号级限流,锁定账号,更新 Redis 订单/任务追踪器与生成冷却; - book 时 429:档期提交频控或出口 IP 限制,不锁定账号,移除任务以轮换代理或重试; - session expired / 401:会话失效,移除任务; - 403 Forbidden / WAF:判定为 IP 受阻,移除任务以轮换代理; - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。 """ biz_err = self._normalize_error(err) acc = None if plg_cfg and plg_cfg.account: acc = plg_cfg.account elif task and getattr(task.instance, "config", None) and task.instance.config.account: acc = task.instance.config.account # 1. 目标站限流 / 429 if isinstance(biz_err, RateLimiteddError): if action in ("spawn", "login", "create_session"): backoff = float(self.m_cfg.login_backoff) self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).") if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0): try: CloudApi.Instance().lock_account(acc.id, int(backoff)) self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.") except Exception as lock_err: self._log(f"Failed to lock account via cloud API: {lock_err}") effective_task_id = task_id if task_id is not None else (task.task_ref if task else None) if effective_task_id is not None: self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff}) self.m_spawn_cooldown_until = time.time() + backoff if task: self._remove_task(task, "spawn account rate limited") elif action == "book": # 预订提交时 429:接口频控或 IP 限制,绝不锁账号,移除实例以更换代理重试 self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...") if task: self._remove_task(task, "booking rate limited") else: self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.") if task: self._remove_task(task, f"{action} rate limited") # 2. 会话过期 / 401 elif isinstance(biz_err, SessionExpiredOrInvalidError): if action in ("spawn", "login", "create_session"): self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...") self.m_spawn_cooldown_until = time.time() + 5.0 else: self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...") if task: self._remove_task(task, f"{action} session expired") # 3. 权限被拒 / WAF 拦截 / 403 elif isinstance(biz_err, PermissionDeniedError): proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0) self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...") if task: self._remove_task(task, f"{action} waf blocked") if action == "spawn": self.m_spawn_cooldown_until = time.time() + proxy_cd # 4. 网络或浏览器超时 elif isinstance(biz_err, NetworkTimeoutError): self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.") if task: task.next_run = time.time() + 5.0 # 5. 兜底策略 else: self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})") if action == "spawn": self.m_spawn_cooldown_until = time.time() + 10.0 elif task: task.next_run = time.time() + 10.0 def _is_within_active_hours(self) -> bool: """判断当前是否在允许创建实例的时间段内""" start_str = self.m_cfg.active_time_start end_str = self.m_cfg.active_time_end current_bj_time = datetime.utcnow().time() start_time = datetime.strptime(start_str, "%H:%M").time() end_time = datetime.strptime(end_str, "%H:%M").time() return start_time <= current_bj_time <= end_time def _maintain_loop(self): self._log("Maintain loop started.") while not self.m_stop_event.is_set(): try: time.sleep(1.0) with self.m_lock: tasks_to_check = list(self.m_tasks) if not tasks_to_check: continue dead_tasks = [] healthy_tasks = [] now = time.time() for t in tasks_to_check: if now >= t.next_remote_ping: t.instance.keep_alive() if t.instance.health_check(): healthy_tasks.append(t) t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5) else: dead_tasks.append(t) else: healthy_tasks.append(t) self._on_maintain_ping(healthy_tasks, dead_tasks) if dead_tasks: with self.m_lock: current_tasks = list(self.m_tasks) self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks] for t in dead_tasks: if t in current_tasks: self._cleanup_task(t, "unhealthy or keep-alive failed") else: with self.m_lock: self.m_tasks = [t for t in self.m_tasks if t in healthy_tasks] except Exception as e: self._log(f'Maintain loop exception: {e}') def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]): """子类维护循环中对于健康/异常 Task 的额外处理""" pass def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool: """判断 query_result 中的可用日期是否在 task 意向范围内(默认 True,Order 模式重写)""" return True def _booking_trigger_loop(self): self._log("Trigger loop started.") while not self.m_stop_event.is_set(): try: time.sleep(0.1) now = time.time() for apt_type in self.m_cfg.appointment_types: redis_key = self._get_redis_key(apt_type.routing_key) raw_data = self.redis_client.get(redis_key) if not raw_data: continue try: data = json.loads(raw_data) query_result = QueryResult.model_validate(data['query_result']) query_result.apt_type = AppointmentType.model_validate(data['apt_type']) except Exception as parse_err: self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.") self.redis_client.delete(redis_key) continue matching_tasks = [] with self.m_lock: for task in self.m_tasks: if now < task.next_run or not task.book_allowed: continue if apt_type.routing_key not in task.acceptable_routing_keys: continue if not self._is_date_of_interest(task, query_result): continue task.next_run = now + self.m_cfg.booker.booking_cooldown matching_tasks.append(task) if matching_tasks: threads = [] for task in matching_tasks: self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}") t = threading.Thread(target=self._execute_book_job, args=(task, query_result)) threads.append(t) t.start() for t in threads: t.join() except Exception as e: self._log(f"Booking trigger loop exception: {e}") def _execute_book_job(self, task: Task, query_result: QueryResult): raise NotImplementedError def _creator_loop(self): raise NotImplementedError def _push_success_notification(self, task_id, order_id, book_res): """成功落单后同步云端与推送微信通知的公共方法""" grab_info = { "account": book_res.account, "session_id": book_res.session_id, "urn": book_res.urn, "slot_date": book_res.book_date, "slot_time": book_res.book_time, "timestamp": int(time.time()), "payment_link": book_res.payment_link } def _update_cloud_success(): try: CloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info}) except Exception as e: self._log(f"Failed to update success state to cloud: {e}") ThreadPool.getInstance().enqueue(_update_cloud_success) notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log) self.redis_client.zrem(self.m_tracker_key, task_id) class BuiltinBookerGCO(BaseBookerGCO): """ 非绑定模式 (公共内置账号池): - 只维护全局 target_instances 数量的实例。 - 所有实例热机等待,发现信号后临时去云端 Pop 订单。 """ TAG = "BUILTIN-BOOKER" def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue with self.m_lock: current = len(self.m_tasks) target = self.m_cfg.booker.target_instances if current < target: self._spawn_worker() def _spawn_worker(self): instance = None task = None success = False plg_cfg = None try: plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life if self.m_cfg.need_account: acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd) plg_cfg.account = type(plg_cfg.account)(**acc) if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types] task = Task( instance=instance, next_run=time.time(), task_ref=None, acceptable_routing_keys=all_keys, source_queue="built-in", book_allowed=True, next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5) ) with self.m_lock: self.m_tasks.append(task) success = True self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed") def _execute_book_job(self, task: Task, query_result: QueryResult): queue_name = f"auto.{query_result.apt_type.routing_key}" task_id = None task_data = None try: task_data = CloudApi.Instance().get_vas_task_pop(queue_name) if not task_data: return task_id = task_data['id'] order_id = task_data.get('order_id') self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0}) user_input = task_data.get('user_inputs', {}) book_res = task.instance.book(query_result, user_input) if book_res.success: self._log(f"✅ BOOK SUCCESS! Order: {order_id}") self._push_success_notification(task_id, order_id, book_res) task.successful_bookings += 1 max_b = self.m_cfg.booker.max_bookings_per_account if max_b > 0 and task.successful_bookings >= max_b: self._log(f"Account reached max bookings ({max_b}). Destroying instance.") self._remove_task(task, "max bookings reached") else: self._log(f"❌ BOOK FAILED for Order: {order_id}") except Exception as e: self._dispatch_error_policy("book", e, task=task) class OrderBookerGCO(BaseBookerGCO): """ 绑定模式 (订单自带账号): - 按城市队列维护热机配额。 - 绝对的 1 对 1 关系:一个实例绑定一个云端订单。 - 预订成功后,实例立即销毁。 """ TAG = "ORDER-BOOKER" def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): super().__init__(cfg, redis_conf, logger) self.m_task_data_cache: Dict[str, dict] = {} self.heartbeat_ttl = 2 * 60.0 def _start_additional_threads(self): threading.Thread(target=self._cache_refresh_loop, daemon=True).start() def _on_task_removed(self, task: Task): task_id = task.task_ref if task_id: self.m_task_data_cache.pop(str(task_id), None) def _on_all_tasks_cleaned(self): self.m_task_data_cache.clear() def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]): if healthy_tasks: new_deadline = time.time() + self.heartbeat_ttl mapping = {str(t.task_ref): new_deadline for t in healthy_tasks} self.redis_client.bulk_zadd(self.m_tracker_key, mapping) if dead_tasks: mapping = {str(t.task_ref): 0 for t in dead_tasks} self.redis_client.bulk_zadd(self.m_tracker_key, mapping) def _cache_refresh_loop(self): self._log("Cache refresh loop started.") refresh_interval = 15 * 60 while not self.m_stop_event.is_set(): try: time.sleep(1) with self.m_lock: tasks_to_check = { tid: data.get('_last_refresh', 0) for tid, data in self.m_task_data_cache.items() } if not tasks_to_check: continue now = time.time() for tid, last_refresh in tasks_to_check.items(): if now - last_refresh >= refresh_interval: fresh_data = CloudApi.Instance().get_vas_task(tid) if fresh_data: fresh_data['_last_refresh'] = time.time() with self.m_lock: if tid in self.m_task_data_cache: self.m_task_data_cache[tid] = fresh_data time.sleep(0.5) except Exception as e: self._log(f'Cache refresh loop exception: {e}') def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool: if query_result.availability_status != AvailabilityStatus.Available: return True task_id = task.task_ref task_data = self.m_task_data_cache.get(str(task_id), {}) user_input = task_data.get('user_inputs', {}) expected_end_date = ( user_input.get('expected_end_date') or '2100-01-01' ) available_date = query_result.earliest_date dt = available_date.strftime("%Y-%m-%d") return dt <= expected_end_date def _execute_book_job(self, task: Task, query_result: QueryResult): task_id = task.task_ref task_data = None try: with self.m_lock: task_data = self.m_task_data_cache.get(str(task_id)) if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']: self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.") self._remove_task(task, "bound task no longer valid") self.redis_client.zrem(self.m_tracker_key, task_id) return order_id = task_data.get('order_id') user_input = task_data.get('user_inputs', {}) book_res = task.instance.book(query_result, user_input) if book_res.success: self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.") self._push_success_notification(task_id, order_id, book_res) self._remove_task(task, "booking success") else: self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.") except Exception as e: self._dispatch_error_policy("book", e, task=task) def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue for apt in self.m_cfg.appointment_types: r_key = apt.routing_key with self.m_lock: active = sum(1 for t in self.m_tasks if t.source_queue == r_key) target = self.m_cfg.booker.target_instances if active < target: self._spawn_worker(r_key) def _spawn_worker(self, target_routing_key: str): instance = None task = None success = False task_id = None try: queue_name = f"auto.{target_routing_key}" task_data = CloudApi.Instance().get_vas_task_pop(queue_name) if not task_data: return task_id = task_data['id'] with self.m_lock: self.m_task_data_cache[str(task_id)] = task_data self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0}) user_inputs = task_data.get('user_inputs', {}) plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life plg_cfg.account.username = user_inputs.get("username", "") plg_cfg.account.password = user_inputs.get("password", "") if not plg_cfg.account.username: return acceptable_keys = [target_routing_key] if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() task = Task( instance=instance, next_run=time.time(), task_ref=task_id, acceptable_routing_keys=acceptable_keys, source_queue=target_routing_key, book_allowed=True, next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5) ) with self.m_lock: self.m_tasks.append(task) success = True self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed") if not success and task_id: with self.m_lock: self.m_task_data_cache.pop(str(task_id), None)