import os from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads import time import json import threading import random from datetime import datetime, timedelta, timezone from typing import List, Dict, Callable, Optional from core_types import ( GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult, AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str, BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError, NetworkTimeoutError, BizLogicError, AccountStateError, AccountAuthError, AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error ) from plugin_factory import PluginFactory from toolkit.cloud_api import CloudApi from toolkit.proxy_manager import ProxyManager from toolkit.task_executor import TaskExecutor from utils.booking_outbox import BookingOutbox from toolkit.notifier import notify_book_result from utils.safe_redis_cli import SafeRedisClient class BaseBookerGCO: """ Base Booker class encapsulating shared infrastructure, lifecycle management, and logic loops. """ TAG = "BOOKER" def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): self.m_cfg = cfg self.m_factory = PluginFactory() self.m_logger = logger self.m_tasks: List[Task] = [] self.m_lock = threading.RLock() self.m_stop_event = threading.Event() self.redis_client = SafeRedisClient(redis_conf, self.m_logger) init_lifecycle(self) self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}" self.m_last_group_query_time = 0.0 self.m_spawn_cooldown_until = 0.0 self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking") self._leased_orders = set() self._pending_results = {} self._outbox = BookingOutbox() self._background_threads = [] self._heartbeat_stop_event = threading.Event() def _log(self, message): prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]" if self.m_logger: self.m_logger(f'{prefix} {message}') else: print(f'{prefix} {message}') def get_status(self) -> GroupStatus: with self.m_lock: tasks = list(self.m_tasks) gid = self.m_cfg.identifier enable = self.m_cfg.enable plugin_name = self.m_cfg.plugin_config.plugin_name plugins_status = [] healthy_count = 0 for t in tasks: if t and t.instance: st = t.instance.get_plugin_status() if st.is_healthy: healthy_count += 1 plugins_status.append(st) return GroupStatus( group_id=gid, enable=enable, plugin_name=plugin_name, total_tasks=len(tasks), healthy_tasks=healthy_count, plugins=plugins_status, updated_at=utc_now_str(), ) @start_once def start(self): if not self.m_cfg.enable: return self._log(f"Starting {self.TAG}...") plugin_name = self.m_cfg.plugin_config.plugin_name class_name = "".join(part.title() for part in plugin_name.split('_')) plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin) self.m_factory.register_plugin(plugin_name, plugin_path, class_name) for target in (self._booking_trigger_loop, self._creator_loop, self._maintain_loop, self._heartbeat_loop, self._success_sync_loop): thread = threading.Thread(target=target, daemon=True, name=target.__name__) self._background_threads.append(thread) thread.start() self._start_additional_threads() def _start_additional_threads(self): """Hook for subclasses to start additional background threads.""" pass @serialized_stop def stop(self): self._log("Stopping Booker...") self.m_stop_event.set() deadline = time.monotonic() + self.m_cfg.shutdown_timeout self._executor.shutdown(timeout=max(0, deadline - time.monotonic())) workers = [t for t in self._background_threads if t.name != '_heartbeat_loop'] join_threads(workers, max(0, deadline - time.monotonic())) with self.m_lock: for task in self.m_tasks: task.removed = True self._resources.track(task.instance) self._resources.close(self.m_cfg.cleanup_timeout) threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1) if threads_done and not self._pending_results: self._heartbeat_stop_event.set() threads_done = join_threads( [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2) if not self._resources.empty or not threads_done or self._pending_results: raise RuntimeError('Booker shutdown incomplete; resources retained for retry') with self.m_lock: self.m_tasks.clear() self._on_all_tasks_cleaned() self._leased_orders.clear() self.redis_client.close() def update_config(self, new_cfg: GroupConfig): """Dynamically updates the configuration.""" with self.m_lock: if self.m_cfg.enable and not new_cfg.enable: self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.") elif not self.m_cfg.enable and new_cfg.enable: self._log("Config dynamically updated: Group ENABLED.") else: self._log("Config dynamically updated: Parameters refreshed.") self.m_cfg = new_cfg def _cleanup_instance(self, instance, reason: str = "", task_ref=None): if instance is None: return True cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout) if not cleaned: self._log(f"Cleanup retained for retry ({reason})") return cleaned def _cleanup_task(self, task: Task, reason: str = ""): if task and task.instance: with task._operation_lock: self._cleanup_instance(task.instance, reason, task.task_ref) def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True): removed = False with self.m_lock: if task in self.m_tasks: task.removed = True self.m_tasks.remove(task) removed = True if task.task_ref is not None and str(task.task_ref) not in self._pending_results: self._leased_orders.discard(str(task.task_ref)) self._on_task_removed(task) if cleanup and removed: self._cleanup_task(task, reason) return removed def _on_task_removed(self, task: Task): """Hook for subclasses when a task is removed (e.g. cleaning up caches).""" pass def _cleanup_all_tasks(self, reason: str = ""): with self.m_lock: tasks = list(self.m_tasks) self.m_tasks.clear() for task in tasks: task.removed = True if task.task_ref is not None and str(task.task_ref) not in self._pending_results: self._leased_orders.discard(str(task.task_ref)) self._on_all_tasks_cleaned() for task in tasks: self._cleanup_task(task, reason) def _on_all_tasks_cleaned(self): """Hook for subclasses when all tasks are cleared.""" pass def _get_redis_key(self, routing_key: str) -> str: return f"vs:signal:{routing_key}" def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None: """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues.""" if acc and acc.id > 0 and acc.username not in ("", "Guest"): try: CloudApi.Instance().lock_account(acc.id, 0) self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).") except Exception as e: self._log(f"Failed to release account [{acc.username}]: {e}") def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None: """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data.""" if not acc or acc.id <= 0 or acc.username in ("", "Guest"): return try: cloud = CloudApi.Instance() url = f"{cloud.base_url}/api/account/update" params = {"account_id": acc.id} far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z") payload = { "status": "disable", "next_use_time": far_future, "extra_data": { "disable_reason": reason, "disabled_at": int(time.time()), "module": self.TAG, "group": getattr(self.m_cfg, "identifier", "") } } cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10) self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}") except Exception as e: self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...") try: CloudApi.Instance().lock_account(acc.id, 86400 * 3650) except Exception as lock_err: self._log(f"Fallback 10-year lock also failed: {lock_err}") def _dispatch_error_policy( self, action: str, err: Exception, task: Optional[Task] = None, plg_cfg: Optional[PluginConfig] = None, task_id: Optional[int] = None ) -> None: """ Booker error dispatch and self-healing policy engine: Executes self-healing actions based on the (action, error type) matrix: - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account or fails order task, never releases back to pool. - 429 during spawn/login: Account-level rate limit. Locks account, updates tracker & cooldown. - 429 during book: Booking slot rate limit or egress IP throttle. Preserves account, removes task to rotate proxy. - 401 / session expired: Session invalidated. Removes task. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn. - Handles logging and timer backoffs internally. """ biz_err = normalize_error(err) active_plg_cfg = plg_cfg or (task.instance.config if task else None) acc = active_plg_cfg.account if active_plg_cfg else None # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found) if isinstance(biz_err, AccountStateError): self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})") effective_task_id = task_id if task_id is not None else (task.task_ref if task else None) # If bound order task, update VAS task status to failed if effective_task_id is not None: try: CloudApi.Instance().update_vas_task(str(effective_task_id), { "status": "failed", "fail_reason": f"Account fatal error: {biz_err.message} (code={biz_err.code})" }) self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}") except Exception as update_err: self._log(f"Failed to update VAS task status: {update_err}") else: self.redis_client.zrem(self.m_tracker_key, effective_task_id) # If pool account, permanently disable it in cloud if acc and acc.id > 0 and acc.username not in ("", "Guest"): self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}") # Never release account back to pool if task: self._remove_task(task, f"account fatal error: {biz_err.message}") if action in ("spawn", "login", "create_session"): self.m_spawn_cooldown_until = time.time() + 5.0 # 1. Target site rate limit / 429 elif isinstance(biz_err, RateLimiteddError): if action in ("spawn", "login", "create_session"): backoff = float(self.m_cfg.login_backoff) self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).") if acc and acc.username not in ("", "Guest") and acc.id > 0: try: CloudApi.Instance().lock_account(acc.id, int(backoff)) self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.") except Exception as lock_err: self._log(f"Failed to lock account via cloud API: {lock_err}") effective_task_id = task_id if task_id is not None else (task.task_ref if task else None) if effective_task_id is not None: self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff}) self.m_spawn_cooldown_until = time.time() + backoff if task: self._remove_task(task, "spawn account rate limited") elif action == "book": # 429 during booking: slot rate limit or IP restriction, preserve account and remove task to rotate proxy self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...") if task: self._remove_task(task, "booking rate limited") else: self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.") if task: self._remove_task(task, f"{action} rate limited") # 2. Session expired / 401 elif isinstance(biz_err, SessionExpiredOrInvalidError): if action in ("spawn", "login", "create_session"): self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...") self.m_spawn_cooldown_until = time.time() + 5.0 else: self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...") if task: self._remove_task(task, f"{action} session expired") # 3. Permission denied / WAF blocked / 403 elif isinstance(biz_err, PermissionDeniedError): proxy_cd = float(self.m_cfg.proxy_cd or 60.0) self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...") if task: self._remove_task(task, f"{action} waf blocked") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="WAF Blocked") self.m_spawn_cooldown_until = time.time() + proxy_cd # 4. Network or browser timeout elif isinstance(biz_err, NetworkTimeoutError): self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.") if task: task.next_run = time.time() + 5.0 if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="Network Timeout") spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)) self.m_spawn_cooldown_until = time.time() + spawn_cd # 5. Fallback policy else: self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}") spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)) self.m_spawn_cooldown_until = time.time() + spawn_cd elif task: task.next_run = time.time() + 10.0 # 6. Proxy health metrics recording if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=active_plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action=action, success=False, http_code=biz_err.http_status, error_type=biz_err.message ) def _is_within_active_hours(self) -> bool: """Checks if current time is within allowed operating hours (UTC).""" start_str = self.m_cfg.active_time_start end_str = self.m_cfg.active_time_end if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str: return True current_utc_time = datetime.now(timezone.utc).time() start_time = datetime.strptime(start_str, "%H:%M").time() end_time = datetime.strptime(end_str, "%H:%M").time() if start_time <= end_time: return start_time <= current_utc_time <= end_time else: return current_utc_time >= start_time or current_utc_time <= end_time def _lease_order(self, task_id): with self.m_lock: self._leased_orders.add(str(task_id)) self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120}) def _renew_leases(self): with self.m_lock: if self._leased_orders: self.redis_client.bulk_zadd(self.m_tracker_key, { tid: time.time() + 120 for tid in self._leased_orders}) def _heartbeat_loop(self): # No browser calls here: long OTP/HTTP waits must not stop order leases. while not self._heartbeat_stop_event.wait(10): self._renew_leases() def _sync_successes(self): try: with self.m_lock: pending = dict(self._pending_results) for task_id, payload in pending.items(): saved = False try: self._outbox.put(task_id, self.m_tracker_key, payload) saved = True except Exception as exc: self._log(f"Local success journal unavailable for task={task_id}: {exc}") saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved if saved: with self.m_lock: self._pending_results.pop(task_id, None) self._leased_orders.discard(task_id) self._outbox.flush(CloudApi.Instance(), self.redis_client._client, self._log, tracker=self.m_tracker_key) except Exception as exc: self._log(f"Success outbox retry failed: {exc}") def _success_sync_loop(self): while not self.m_stop_event.is_set() or self._pending_results: self._sync_successes() if self.m_stop_event.is_set() and not self._pending_results: break if self.m_stop_event.is_set(): self._heartbeat_stop_event.wait(5) else: self.m_stop_event.wait(5) def _maintain_task(self, task): try: with task._operation_lock, self._resources.operation(task.instance): if task.removed or self.m_stop_event.is_set(): return task.instance.keep_alive() if task.instance.health_check(): task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5)) else: self._remove_task(task, "unhealthy or keep-alive failed") except Exception as exc: task.next_remote_ping = time.time() + 5 self._dispatch_error_policy("keep_alive", exc, task=task) finally: with self.m_lock: task.busy = False def _schedule_maintenance(self): with self.m_lock: if self.m_stop_event.is_set(): return for task in self.m_tasks: if task.busy or task.removed or time.time() < task.next_remote_ping: continue task.busy = True if not self._executor.submit(self._maintain_task, task): task.busy = False def _maintain_loop(self): while not self.m_stop_event.wait(1): self._schedule_maintenance() def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool: """Checks if available dates in query_result match task preferences (defaults to True).""" return True def _run_book_job(self, task, query_result): try: with task._operation_lock, self._resources.operation(task.instance): if not task.removed and not self.m_stop_event.is_set(): self._execute_book_job(task, query_result) finally: with self.m_lock: task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown) task.busy = False def _poll_booking_signals(self): now = time.time() with self.m_lock: ready = [t for t in self.m_tasks if not t.busy and not t.removed and t.book_allowed and now >= t.next_run] keys = {key for t in ready for key in t.acceptable_routing_keys} apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys] if not apt_types: return redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types] signals = self.redis_client.mget(redis_keys) for apt_type, raw_data in zip(apt_types, signals): if not raw_data: continue try: data = json.loads(raw_data) result = QueryResult.model_validate(data['query_result']) result.apt_type = AppointmentType.model_validate(data['apt_type']) if result.apt_type.routing_key != apt_type.routing_key: raise ValueError('signal routing key mismatch') except Exception as exc: # A producer may have replaced this value since MGET. Do not # delete a newer signal based on an old malformed payload. self._log(f"Invalid signal for {apt_type.routing_key}: {exc}") continue with self.m_lock: for task in ready: if (task.busy or task.removed or self.m_stop_event.is_set() or apt_type.routing_key not in task.acceptable_routing_keys or not self._is_date_of_interest(task, result)): continue task.busy = True if not self._executor.submit(self._run_book_job, task, result): task.busy = False def _booking_trigger_loop(self): self._log("Trigger loop started.") while not self.m_stop_event.is_set(): try: if self.m_cfg.enable and self._is_within_active_hours(): self._poll_booking_signals() except Exception as exc: self._log(f"Booking trigger loop exception: {exc}") self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval) def _execute_book_job(self, task: Task, query_result: QueryResult): raise NotImplementedError def _creator_loop(self): raise NotImplementedError def _push_success_notification(self, task_id, order_id, book_res): """Shared helper to sync booking success state to cloud and dispatch notifications.""" grab_info = { "account": book_res.account, "session_id": book_res.session_id, "urn": book_res.urn, "slot_date": book_res.book_date, "slot_time": book_res.book_time, "timestamp": int(time.time()), "payment_link": book_res.payment_link } payload = {"status": "grabbed", "grabbed_history": grab_info} # Commit locally before retiring the browser or dropping its lease. with self.m_lock: self._pending_results[str(task_id)] = payload self._sync_successes() try: notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log) except Exception as exc: self._log(f"Booking succeeded, but notification could not be queued: {exc}") class BuiltinBookerGCO(BaseBookerGCO): """ Unbound mode (shared built-in account pool): - Maintains target_instances count of instances globally. - Instances stay warm; when a slot signal is detected, orders are popped on-demand from the cloud queue. """ TAG = "BUILTIN-BOOKER" def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue with self.m_lock: current = len(self.m_tasks) target = self.m_cfg.booker.target_instances if current < target: self._spawn_worker() def _spawn_worker(self): if self.m_stop_event.is_set() or not self._resources.retry_failed(): return instance = None task = None success = False plg_cfg = None try: plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life if self.m_cfg.need_account: acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd) plg_cfg.account = type(plg_cfg.account)(**acc) if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy( self.m_cfg.proxy_pool, self.m_cfg.proxy_cd, plugin_name=self.m_cfg.plugin_config.plugin_name ) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) self._resources.track(instance) with self._resources.operation(instance): instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types] task = Task( instance=instance, next_run=time.time(), task_ref=None, acceptable_routing_keys=all_keys, source_queue="built-in", book_allowed=True, next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5) ) with self.m_lock: if self.m_stop_event.is_set(): return self.m_tasks.append(task) success = True if plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="spawn", success=True, http_code=200 ) self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed") def _execute_book_job(self, task: Task, query_result: QueryResult): queue_name = f"auto.{query_result.apt_type.routing_key}" task_data = None try: task_data = CloudApi.Instance().get_vas_task_pop(queue_name) except BizLogicError: # Queue has no pending tasks, return normally return except Exception as e: self._log(f"⚠️ [CloudApi] Failed to pop task from {queue_name}: {e}") return if not task_data: return task_id = task_data.get('id') order_id = task_data.get('order_id') self._lease_order(task_id) book_res = None try: user_input = task_data.get('user_inputs', {}) book_res = task.instance.book(query_result, user_input) if book_res.success: self._log(f"✅ BOOK SUCCESS! Order: {order_id}") self._push_success_notification(task_id, order_id, book_res) task.successful_bookings += 1 max_b = self.m_cfg.booker.max_bookings_per_account if max_b > 0 and task.successful_bookings >= max_b: self._log(f"Account reached max bookings ({max_b}). Destroying instance.") self._remove_task(task, "max bookings reached") else: self._log(f"❌ BOOK FAILED for Order: {order_id}") except Exception as e: self._dispatch_error_policy("book", e, task=task, task_id=task_id) finally: with self.m_lock: if str(task_id) not in self._pending_results: self._leased_orders.discard(str(task_id)) if book_res is None or not book_res.success: self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()}) class OrderBookerGCO(BaseBookerGCO): """ Bound mode (order provides its own account credentials): - Maintains warm-up quotas per routing queue. - Strict 1:1 binding: one instance per cloud order. - Instance is destroyed immediately after booking completes. """ TAG = "ORDER-BOOKER" def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): super().__init__(cfg, redis_conf, logger) self.m_task_data_cache: Dict[str, dict] = {} def _start_additional_threads(self): thread = threading.Thread(target=self._cache_refresh_loop, daemon=True) self._background_threads.append(thread) thread.start() def _on_task_removed(self, task: Task): task_id = task.task_ref if task_id is not None: self.m_task_data_cache.pop(str(task_id), None) def _on_all_tasks_cleaned(self): self.m_task_data_cache.clear() def _cache_refresh_loop(self): self._log("Cache refresh loop started.") refresh_interval = 15 * 60 while not self.m_stop_event.is_set(): try: if self.m_stop_event.wait(1): break with self.m_lock: tasks_to_check = { tid: data.get('_last_refresh', 0) for tid, data in self.m_task_data_cache.items() } if not tasks_to_check: continue now = time.time() for tid, last_refresh in tasks_to_check.items(): if self.m_stop_event.is_set(): break if now - last_refresh >= refresh_interval: fresh_data = CloudApi.Instance().get_vas_task(tid) if fresh_data: fresh_data['_last_refresh'] = time.time() with self.m_lock: if tid in self.m_task_data_cache: self.m_task_data_cache[tid] = fresh_data self.m_stop_event.wait(0.5) except Exception as e: self._log(f'Cache refresh loop exception: {e}') def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool: if query_result.availability_status != AvailabilityStatus.Available: return True task_id = task.task_ref task_data = self.m_task_data_cache.get(str(task_id), {}) user_input = task_data.get('user_inputs', {}) expected_end_date = ( user_input.get('expected_end_date') or '2100-01-01' ) available_date = query_result.earliest_date if available_date is None: return False dt = available_date.strftime("%Y-%m-%d") return dt <= expected_end_date def _execute_book_job(self, task: Task, query_result: QueryResult): task_id = task.task_ref task_data = None try: with self.m_lock: task_data = self.m_task_data_cache.get(str(task_id)) if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']: self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.") self._remove_task(task, "bound task no longer valid") self.redis_client.zrem(self.m_tracker_key, task_id) return order_id = task_data.get('order_id') user_input = task_data.get('user_inputs', {}) book_res = task.instance.book(query_result, user_input) if book_res.success: self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.") self._push_success_notification(task_id, order_id, book_res) self._remove_task(task, "booking success") else: self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.") except Exception as e: self._dispatch_error_policy("book", e, task=task) def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue for apt in self.m_cfg.appointment_types: r_key = apt.routing_key with self.m_lock: active = sum(1 for t in self.m_tasks if t.source_queue == r_key) target = self.m_cfg.booker.target_instances if active < target: self._spawn_worker(r_key) def _spawn_worker(self, target_routing_key: str): if self.m_stop_event.is_set() or not self._resources.retry_failed(): return instance = None task = None success = False task_id = None plg_cfg = None try: queue_name = f"auto.{target_routing_key}" try: task_data = CloudApi.Instance().get_vas_task_pop(queue_name) except BizLogicError: # Queue has no pending tasks, return directly return if not task_data: return task_id = task_data['id'] with self.m_lock: self.m_task_data_cache[str(task_id)] = task_data self._lease_order(task_id) user_inputs = task_data.get('user_inputs', {}) plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life plg_cfg.account.username = user_inputs.get("username", "") plg_cfg.account.password = user_inputs.get("password", "") if not plg_cfg.account.username: raise AccountIncompleteError(f"Task={task_id} missing username credentials in user_inputs") acceptable_keys = [target_routing_key] if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy( self.m_cfg.proxy_pool, self.m_cfg.proxy_cd, plugin_name=self.m_cfg.plugin_config.plugin_name ) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) self._resources.track(instance) with self._resources.operation(instance): instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() task = Task( instance=instance, next_run=time.time(), task_ref=task_id, acceptable_routing_keys=acceptable_keys, source_queue=target_routing_key, book_allowed=True, next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5) ) with self.m_lock: if self.m_stop_event.is_set(): return self.m_tasks.append(task) success = True if plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="spawn", success=True, http_code=200 ) self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed") if not success and task_id is not None: with self.m_lock: self.m_task_data_cache.pop(str(task_id), None) try: with self.m_lock: self._leased_orders.discard(str(task_id)) # Retain a recovery deadline (including any 429 cooldown). self.redis_client.zadd(self.m_tracker_key, { str(task_id): max(time.time(), self.m_spawn_cooldown_until)}) except Exception: pass