| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899 |
- import os
- from copy import deepcopy
- from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
- import time
- import json
- import threading
- import random
- from datetime import datetime, timedelta, timezone
- from typing import List, Dict, Callable, Optional
- from core_types import (
- GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult,
- AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
- BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
- NetworkTimeoutError, BizLogicError, AccountStateError, AccountAuthError,
- AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error
- )
- from plugin_factory import PluginFactory
- from toolkit.cloud_api import CloudApi
- from toolkit.proxy_manager import ProxyManager
- from toolkit.task_executor import TaskExecutor
- from utils.booking_outbox import BookingOutbox
- from toolkit.notifier import notify_book_result
- from utils.safe_redis_cli import SafeRedisClient
- class BaseBookerGCO:
- """
- Base Booker class encapsulating shared infrastructure, lifecycle management, and logic loops.
- """
- TAG = "BOOKER"
- def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
- self.m_cfg = cfg
- self.m_factory = PluginFactory()
- self.m_logger = logger
- self.m_tasks: List[Task] = []
- self.m_lock = threading.RLock()
- self.m_stop_event = threading.Event()
- self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
- init_lifecycle(self)
- self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
- self.m_last_group_query_time = 0.0
- self.m_spawn_cooldown_until = 0.0
- self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking")
- self._leased_orders = set()
- self._pending_results = {}
- self._outbox = BookingOutbox()
- self._background_threads = []
- self._heartbeat_stop_event = threading.Event()
- def _log(self, message):
- prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
- if self.m_logger:
- self.m_logger(f'{prefix} {message}')
- else:
- print(f'{prefix} {message}')
- def get_status(self) -> GroupStatus:
- with self.m_lock:
- tasks = list(self.m_tasks)
- gid = self.m_cfg.identifier
- enable = self.m_cfg.enable
- plugin_name = self.m_cfg.plugin_config.plugin_name
- plugins_status = []
- healthy_count = 0
- for t in tasks:
- if t and t.instance:
- st = t.instance.get_plugin_status()
- if st.is_healthy:
- healthy_count += 1
- plugins_status.append(st)
- return GroupStatus(
- group_id=gid,
- enable=enable,
- plugin_name=plugin_name,
- total_tasks=len(tasks),
- healthy_tasks=healthy_count,
- plugins=plugins_status,
- updated_at=utc_now_str(),
- )
- @start_once
- def start(self):
- if not self.m_cfg.enable:
- return
- self._log(f"Starting {self.TAG}...")
- plugin_name = self.m_cfg.plugin_config.plugin_name
- class_name = "".join(part.title() for part in plugin_name.split('_'))
- plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
- self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
- if plugin_name:
- self.m_factory.ensure_booking_supported(plugin_name)
- for target in (self._booking_trigger_loop, self._creator_loop,
- self._maintain_loop, self._heartbeat_loop, self._success_sync_loop):
- thread = threading.Thread(target=target, daemon=True, name=target.__name__)
- self._background_threads.append(thread)
- thread.start()
- self._start_additional_threads()
- def _start_additional_threads(self):
- """Hook for subclasses to start additional background threads."""
- pass
- @serialized_stop
- def stop(self):
- self._log("Stopping Booker...")
- self.m_stop_event.set()
- deadline = time.monotonic() + self.m_cfg.shutdown_timeout
- self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
- workers = [t for t in self._background_threads if t.name != '_heartbeat_loop']
- join_threads(workers, max(0, deadline - time.monotonic()))
- with self.m_lock:
- for task in self.m_tasks:
- task.removed = True
- self._resources.track(task.instance)
- self._resources.close(self.m_cfg.cleanup_timeout)
- threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1)
- if threads_done and not self._pending_results:
- self._heartbeat_stop_event.set()
- threads_done = join_threads(
- [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2)
- if not self._resources.empty or not threads_done or self._pending_results:
- raise RuntimeError('Booker shutdown incomplete; resources retained for retry')
- with self.m_lock:
- self.m_tasks.clear()
- self._on_all_tasks_cleaned()
- self._leased_orders.clear()
- self.redis_client.close()
- def update_config(self, new_cfg: GroupConfig):
- """Dynamically updates the configuration."""
- with self.m_lock:
- if self.m_cfg.enable and not new_cfg.enable:
- self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
- elif not self.m_cfg.enable and new_cfg.enable:
- self._log("Config dynamically updated: Group ENABLED.")
- else:
- self._log("Config dynamically updated: Parameters refreshed.")
-
- self.m_cfg = new_cfg
- def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
- if instance is None:
- return True
- cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
- if not cleaned:
- self._log(f"Cleanup retained for retry ({reason})")
- return cleaned
- def _cleanup_task(self, task: Task, reason: str = ""):
- if task and task.instance:
- with task._operation_lock:
- self._cleanup_instance(task.instance, reason, task.task_ref)
- def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
- removed = False
- with self.m_lock:
- if task in self.m_tasks:
- task.removed = True
- self.m_tasks.remove(task)
- removed = True
- if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
- self._leased_orders.discard(str(task.task_ref))
- self._on_task_removed(task)
-
- if cleanup and removed:
- self._cleanup_task(task, reason)
- return removed
- def _on_task_removed(self, task: Task):
- """Hook for subclasses when a task is removed (e.g. cleaning up caches)."""
- pass
- def _cleanup_all_tasks(self, reason: str = ""):
- with self.m_lock:
- tasks = list(self.m_tasks)
- self.m_tasks.clear()
- for task in tasks:
- task.removed = True
- if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
- self._leased_orders.discard(str(task.task_ref))
- self._on_all_tasks_cleaned()
- for task in tasks:
- self._cleanup_task(task, reason)
- def _on_all_tasks_cleaned(self):
- """Hook for subclasses when all tasks are cleared."""
- pass
- def _get_redis_key(self, routing_key: str) -> str:
- return f"vs:signal:{routing_key}"
- def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
- """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
- if acc and acc.id > 0 and acc.username not in ("", "Guest"):
- try:
- CloudApi.Instance().lock_account(acc.id, 0)
- self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
- except Exception as e:
- self._log(f"Failed to release account [{acc.username}]: {e}")
- def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
- """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
- if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
- return
- try:
- cloud = CloudApi.Instance()
- url = f"{cloud.base_url}/api/account/update"
- params = {"account_id": acc.id}
- far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
- payload = {
- "status": "disable",
- "next_use_time": far_future,
- "extra_data": {
- "disable_reason": reason,
- "disabled_at": int(time.time()),
- "module": self.TAG,
- "group": getattr(self.m_cfg, "identifier", "")
- }
- }
- cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
- self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
- except Exception as e:
- self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
- try:
- CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
- except Exception as lock_err:
- self._log(f"Fallback 10-year lock also failed: {lock_err}")
- def _dispatch_error_policy(
- self,
- action: str,
- err: Exception,
- task: Optional[Task] = None,
- plg_cfg: Optional[PluginConfig] = None,
- task_id: Optional[int] = None
- ) -> None:
- """
- Booker error dispatch and self-healing policy engine:
- Executes self-healing actions based on the (action, error type) matrix:
- - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account or fails order task, never releases back to pool.
- - 429 during spawn/login: Account-level rate limit. Locks account, updates tracker & cooldown.
- - 429 during book: Booking slot rate limit or egress IP throttle. Preserves account, removes task to rotate proxy.
- - 401 / session expired: Session invalidated. Removes task.
- - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
- - Handles logging and timer backoffs internally.
- """
- biz_err = normalize_error(err)
- active_plg_cfg = plg_cfg or (task.instance.config if task else None)
- acc = active_plg_cfg.account if active_plg_cfg else None
- # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
- if isinstance(biz_err, AccountStateError):
- self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
- effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
- # If bound order task, update VAS task status to failed
- if effective_task_id is not None:
- try:
- CloudApi.Instance().update_vas_task(str(effective_task_id), {
- "status": "failed",
- "fail_reason": f"Account fatal error: {biz_err.message} (code={biz_err.code})"
- })
- self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
- except Exception as update_err:
- self._log(f"Failed to update VAS task status: {update_err}")
- else:
- self.redis_client.zrem(self.m_tracker_key, effective_task_id)
- # If pool account, permanently disable it in cloud
- if acc and acc.id > 0 and acc.username not in ("", "Guest"):
- self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
- # Never release account back to pool
- if task:
- self._remove_task(task, f"account fatal error: {biz_err.message}")
- if action in ("spawn", "login", "create_session"):
- self.m_spawn_cooldown_until = time.time() + 5.0
- # 1. Target site rate limit / 429
- elif isinstance(biz_err, RateLimiteddError):
- if action in ("spawn", "login", "create_session"):
- backoff = float(self.m_cfg.login_backoff)
- self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
- if acc and acc.username not in ("", "Guest") and acc.id > 0:
- try:
- CloudApi.Instance().lock_account(acc.id, int(backoff))
- self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
- except Exception as lock_err:
- self._log(f"Failed to lock account via cloud API: {lock_err}")
- effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
- if effective_task_id is not None:
- self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
- self.m_spawn_cooldown_until = time.time() + backoff
- if task:
- self._remove_task(task, "spawn account rate limited")
- elif action == "book":
- # 429 during booking: slot rate limit or IP restriction, preserve account and remove task to rotate proxy
- self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
- if task:
- self._remove_task(task, "booking rate limited")
- else:
- self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
- if task:
- self._remove_task(task, f"{action} rate limited")
- # 2. Session expired / 401
- elif isinstance(biz_err, SessionExpiredOrInvalidError):
- if action in ("spawn", "login", "create_session"):
- self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
- self.m_spawn_cooldown_until = time.time() + 5.0
- else:
- self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
- if task:
- self._remove_task(task, f"{action} session expired")
- # 3. Permission denied / WAF blocked / 403
- elif isinstance(biz_err, PermissionDeniedError):
- proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
- self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
- if task:
- self._remove_task(task, f"{action} waf blocked")
- if action in ("spawn", "login", "create_session"):
- self._release_account_safely(acc, reason="WAF Blocked")
- self.m_spawn_cooldown_until = time.time() + proxy_cd
- # 4. Network or browser timeout
- elif isinstance(biz_err, NetworkTimeoutError):
- self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
- if task:
- task.next_run = time.time() + 5.0
- if action in ("spawn", "login", "create_session"):
- self._release_account_safely(acc, reason="Network Timeout")
- spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
- self.m_spawn_cooldown_until = time.time() + spawn_cd
- # 5. Fallback policy
- else:
- self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
- if action in ("spawn", "login", "create_session"):
- self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
- spawn_cd = max(30.0, float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0))
- self.m_spawn_cooldown_until = time.time() + spawn_cd
- elif task:
- task.next_run = time.time() + 10.0
- # 6. Proxy health metrics recording
- if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
- ProxyManager.Instance().mark_proxy_result(
- proxy=active_plg_cfg.proxy.model_dump(),
- plugin_name=self.m_cfg.plugin_config.plugin_name,
- action=action,
- success=False,
- http_code=biz_err.http_status,
- error_type=biz_err.message
- )
- def _is_within_active_hours(self) -> bool:
- """Checks if current time is within allowed operating hours (UTC)."""
- start_str = self.m_cfg.active_time_start
- end_str = self.m_cfg.active_time_end
- if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
- return True
- current_utc_time = datetime.now(timezone.utc).time()
- start_time = datetime.strptime(start_str, "%H:%M").time()
- end_time = datetime.strptime(end_str, "%H:%M").time()
- if start_time <= end_time:
- return start_time <= current_utc_time <= end_time
- else:
- return current_utc_time >= start_time or current_utc_time <= end_time
- def _lease_order(self, task_id):
- with self.m_lock:
- self._leased_orders.add(str(task_id))
- self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120})
- def _renew_leases(self):
- with self.m_lock:
- if self._leased_orders:
- self.redis_client.bulk_zadd(self.m_tracker_key, {
- tid: time.time() + 120 for tid in self._leased_orders})
- def _heartbeat_loop(self):
- # No browser calls here: long OTP/HTTP waits must not stop order leases.
- while not self._heartbeat_stop_event.wait(10):
- self._renew_leases()
- def _sync_successes(self):
- try:
- with self.m_lock:
- pending = dict(self._pending_results)
- for task_id, payload in pending.items():
- saved = False
- try:
- self._outbox.put(task_id, self.m_tracker_key, payload)
- saved = True
- except Exception as exc:
- self._log(f"Local success journal unavailable for task={task_id}: {exc}")
- saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved
- if saved:
- with self.m_lock:
- self._pending_results.pop(task_id, None)
- self._leased_orders.discard(task_id)
- self._outbox.flush(CloudApi.Instance(), self.redis_client._client,
- self._log, tracker=self.m_tracker_key)
- except Exception as exc:
- self._log(f"Success outbox retry failed: {exc}")
- def _success_sync_loop(self):
- while not self.m_stop_event.is_set() or self._pending_results:
- self._sync_successes()
- if self.m_stop_event.is_set() and not self._pending_results:
- break
- if self.m_stop_event.is_set():
- self._heartbeat_stop_event.wait(5)
- else:
- self.m_stop_event.wait(5)
- def _maintain_task(self, task):
- try:
- with task._operation_lock, self._resources.operation(task.instance):
- if task.removed or self.m_stop_event.is_set():
- return
- task.instance.keep_alive()
- if task.instance.health_check():
- task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5))
- else:
- self._remove_task(task, "unhealthy or keep-alive failed")
- except Exception as exc:
- task.next_remote_ping = time.time() + 5
- self._dispatch_error_policy("keep_alive", exc, task=task)
- finally:
- with self.m_lock:
- task.busy = False
- def _schedule_maintenance(self):
- with self.m_lock:
- if self.m_stop_event.is_set():
- return
- for task in self.m_tasks:
- if task.busy or task.removed or time.time() < task.next_remote_ping:
- continue
- task.busy = True
- if not self._executor.submit(self._maintain_task, task):
- task.busy = False
- def _maintain_loop(self):
- while not self.m_stop_event.wait(1):
- self._schedule_maintenance()
- def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
- """Checks if available dates in query_result match task preferences (defaults to True)."""
- return True
- def _run_book_job(self, task, query_result):
- try:
- with task._operation_lock, self._resources.operation(task.instance):
- if not task.removed and not self.m_stop_event.is_set():
- self._execute_book_job(task, query_result)
- finally:
- with self.m_lock:
- task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown)
- task.busy = False
- def _poll_booking_signals(self):
- now = time.time()
- with self.m_lock:
- ready = [t for t in self.m_tasks if not t.busy and not t.removed
- and t.book_allowed and now >= t.next_run]
- keys = {key for t in ready for key in t.acceptable_routing_keys}
- apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys]
- if not apt_types:
- return
- redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types]
- signals = self.redis_client.mget(redis_keys)
- for apt_type, raw_data in zip(apt_types, signals):
- if not raw_data:
- continue
- try:
- data = json.loads(raw_data)
- result = QueryResult.model_validate(data['query_result'])
- result.apt_type = AppointmentType.model_validate(data['apt_type'])
- if result.apt_type.routing_key != apt_type.routing_key:
- raise ValueError('signal routing key mismatch')
- if result.availability_status not in (AvailabilityStatus.Available, AvailabilityStatus.Waitlist):
- continue
- if data.get('expires_at', float('inf')) <= time.time():
- continue
- except Exception as exc:
- # A producer may have replaced this value since MGET. Do not
- # delete a newer signal based on an old malformed payload.
- self._log(f"Invalid signal for {apt_type.routing_key}: {exc}")
- continue
- with self.m_lock:
- for task in ready:
- if (task.busy or task.removed or self.m_stop_event.is_set()
- or apt_type.routing_key not in task.acceptable_routing_keys
- or not self._is_date_of_interest(task, result)):
- continue
- task.busy = True
- if not self._executor.submit(self._run_book_job, task, result):
- task.busy = False
- def _booking_trigger_loop(self):
- self._log("Trigger loop started.")
- while not self.m_stop_event.is_set():
- try:
- if self.m_cfg.enable and self._is_within_active_hours():
- self._poll_booking_signals()
- except Exception as exc:
- self._log(f"Booking trigger loop exception: {exc}")
- self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval)
- def _execute_book_job(self, task: Task, query_result: QueryResult):
- raise NotImplementedError
- def _creator_loop(self):
- raise NotImplementedError
- def _push_success_notification(self, task_id, order_id, book_res):
- """Shared helper to sync booking success state to cloud and dispatch notifications."""
- grab_info = {
- "order_id": str(order_id) if order_id else None,
- "account": book_res.account,
- "session_id": book_res.session_id,
- "urn": book_res.urn,
- "slot_date": book_res.book_date,
- "slot_time": book_res.book_time,
- "timestamp": int(time.time()),
- "payment_link": book_res.payment_link
- }
-
- payload = {"status": "grabbed", "grabbed_history": grab_info}
- # Commit locally before retiring the browser or dropping its lease.
- with self.m_lock:
- self._pending_results[str(task_id)] = payload
- self._sync_successes()
- try:
- notify_book_result(book_res, order_id=str(order_id) if order_id else None,
- logger=self._log, event_id='booking:order:' + str(task_id))
- except Exception as exc:
- self._log(f"Booking succeeded, but notification could not be queued: {exc}")
- class BuiltinBookerGCO(BaseBookerGCO):
- """
- Unbound mode (shared built-in account pool):
- - Maintains target_instances count of instances globally.
- - Instances stay warm; when a slot signal is detected, orders are popped on-demand from the cloud queue.
- """
- TAG = "BUILTIN-BOOKER"
- def _creator_loop(self):
- self._log("Creator loop started.")
- while not self.m_stop_event.wait(1.0):
- if not self._is_within_active_hours():
- continue
- if time.time() < self.m_spawn_cooldown_until:
- continue
-
- with self.m_lock:
- current = len(self.m_tasks)
- target = self.m_cfg.booker.target_instances
- if current < target:
- self._spawn_worker()
- def _spawn_worker(self):
- if self.m_stop_event.is_set() or not self._resources.retry_failed():
- return
- instance = None
- task = None
- success = False
- plg_cfg = None
-
- try:
- plg_cfg = PluginConfig()
- plg_cfg.debug = self.m_cfg.debug
- plg_cfg.free_config = self.m_cfg.free_config
- plg_cfg.session_max_life = self.m_cfg.session_max_life
- if self.m_cfg.need_account:
- acc = CloudApi.Instance().get_next_account(self.m_cfg.booker.account_pool_id, self.m_cfg.booker.account_cd)
- plg_cfg.account = type(plg_cfg.account)(**acc)
- if self.m_cfg.need_proxy:
- proxy = ProxyManager.Instance().get_next_proxy(
- self.m_cfg.proxy_pool,
- self.m_cfg.proxy_cd,
- plugin_name=self.m_cfg.plugin_config.plugin_name
- )
- plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
- instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
- self._resources.track(instance)
- with self._resources.operation(instance):
- instance.set_log(self.m_logger)
- instance.set_config(plg_cfg)
- instance.create_session()
- all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
- task = Task(
- instance=instance,
- next_run=time.time(),
- task_ref=None,
- acceptable_routing_keys=all_keys,
- source_queue="built-in",
- book_allowed=True,
- next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
- )
- with self.m_lock:
- if self.m_stop_event.is_set():
- return
- self.m_tasks.append(task)
- success = True
- if plg_cfg.proxy.port > 0:
- ProxyManager.Instance().mark_proxy_result(
- proxy=plg_cfg.proxy.model_dump(),
- plugin_name=self.m_cfg.plugin_config.plugin_name,
- action="spawn",
- success=True,
- http_code=200
- )
- self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
- except Exception as e:
- self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
- finally:
- if not success and instance:
- self._cleanup_instance(instance, "spawn failed")
- def _execute_book_job(self, task: Task, query_result: QueryResult):
- queue_name = f"auto.{query_result.apt_type.routing_key}"
- task_data = None
- try:
- task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
- except BizLogicError:
- # Queue has no pending tasks, return normally
- return
- except Exception as e:
- self._log(f"⚠️ [CloudApi] Failed to pop task from {queue_name}: {e}")
- return
- if not task_data:
- return
- task_id = task_data.get('id')
- order_id = task_data.get('order_id')
- self._lease_order(task_id)
- book_res = None
-
- try:
- user_input = task_data.get('user_inputs', {})
- book_res = task.instance.book(query_result, deepcopy(user_input))
-
- if book_res.success:
- self._log(f"✅ BOOK SUCCESS! Order: {order_id}")
- self._push_success_notification(task_id, order_id, book_res)
-
- task.successful_bookings += 1
- max_b = self.m_cfg.booker.max_bookings_per_account
- if max_b > 0 and task.successful_bookings >= max_b:
- self._log(f"Account reached max bookings ({max_b}). Destroying instance.")
- self._remove_task(task, "max bookings reached")
- else:
- self._log(f"❌ BOOK FAILED for Order: {order_id}")
- except Exception as e:
- self._dispatch_error_policy("book", e, task=task, task_id=task_id)
- finally:
- with self.m_lock:
- if str(task_id) not in self._pending_results:
- self._leased_orders.discard(str(task_id))
- if (book_res is None or not book_res.success):
- self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()})
- class OrderBookerGCO(BaseBookerGCO):
- """
- Bound mode (order provides its own account credentials):
- - Maintains warm-up quotas per routing queue.
- - Strict 1:1 binding: one instance per cloud order.
- - Instance is destroyed immediately after booking completes.
- """
- TAG = "ORDER-BOOKER"
- def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
- super().__init__(cfg, redis_conf, logger)
- self.m_task_data_cache: Dict[str, dict] = {}
- def _start_additional_threads(self):
- thread = threading.Thread(target=self._cache_refresh_loop, daemon=True)
- self._background_threads.append(thread)
- thread.start()
- def _on_task_removed(self, task: Task):
- task_id = task.task_ref
- if task_id is not None:
- self.m_task_data_cache.pop(str(task_id), None)
- def _on_all_tasks_cleaned(self):
- self.m_task_data_cache.clear()
- def _cache_refresh_loop(self):
- self._log("Cache refresh loop started.")
- refresh_interval = 15 * 60
-
- while not self.m_stop_event.is_set():
- try:
- if self.m_stop_event.wait(1):
- break
- with self.m_lock:
- tasks_to_check = {
- tid: data.get('_last_refresh', 0)
- for tid, data in self.m_task_data_cache.items()
- }
-
- if not tasks_to_check:
- continue
-
- now = time.time()
- for tid, last_refresh in tasks_to_check.items():
- if self.m_stop_event.is_set():
- break
- if now - last_refresh >= refresh_interval:
- fresh_data = CloudApi.Instance().get_vas_task(tid)
- if fresh_data:
- fresh_data['_last_refresh'] = time.time()
-
- with self.m_lock:
- if tid in self.m_task_data_cache:
- self.m_task_data_cache[tid] = fresh_data
- self.m_stop_event.wait(0.5)
- except Exception as e:
- self._log(f'Cache refresh loop exception: {e}')
- def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
- if query_result.availability_status != AvailabilityStatus.Available:
- return True
- task_id = task.task_ref
- task_data = self.m_task_data_cache.get(str(task_id), {})
- user_input = task_data.get('user_inputs', {})
- expected_end_date = (
- user_input.get('expected_end_date')
- or '2100-01-01'
- )
- available_date = query_result.earliest_date
- if available_date is None:
- return False
- dt = available_date.strftime("%Y-%m-%d")
- return dt <= expected_end_date
- def _execute_book_job(self, task: Task, query_result: QueryResult):
- task_id = task.task_ref
- task_data = None
- try:
- with self.m_lock:
- task_data = self.m_task_data_cache.get(str(task_id))
- if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']:
- self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
- self._remove_task(task, "bound task no longer valid")
- self.redis_client.zrem(self.m_tracker_key, task_id)
- return
-
- order_id = task_data.get('order_id')
- user_input = task_data.get('user_inputs', {})
- book_res = task.instance.book(query_result, deepcopy(user_input))
- if book_res.success:
- self._log(f"✅ BOOK SUCCESS! Order: {order_id}. Destroying instance.")
- self._push_success_notification(task_id, order_id, book_res)
- self._remove_task(task, "booking success")
- else:
- self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
- except Exception as e:
- self._dispatch_error_policy("book", e, task=task)
- def _creator_loop(self):
- self._log("Creator loop started.")
- while not self.m_stop_event.wait(1.0):
- if not self._is_within_active_hours():
- continue
- if time.time() < self.m_spawn_cooldown_until:
- continue
- for apt in self.m_cfg.appointment_types:
- r_key = apt.routing_key
- with self.m_lock:
- active = sum(1 for t in self.m_tasks if t.source_queue == r_key)
- target = self.m_cfg.booker.target_instances
- if active < target:
- self._spawn_worker(r_key)
- def _spawn_worker(self, target_routing_key: str):
- if self.m_stop_event.is_set() or not self._resources.retry_failed():
- return
- instance = None
- task = None
- success = False
- task_id = None
- plg_cfg = None
- try:
- queue_name = f"auto.{target_routing_key}"
- try:
- task_data = CloudApi.Instance().get_vas_task_pop(queue_name)
- except BizLogicError:
- # Queue has no pending tasks, return directly
- return
- if not task_data:
- return
-
- task_id = task_data['id']
-
- with self.m_lock:
- self.m_task_data_cache[str(task_id)] = task_data
-
- self._lease_order(task_id)
- user_inputs = task_data.get('user_inputs', {})
-
- plg_cfg = PluginConfig()
- plg_cfg.debug = self.m_cfg.debug
- plg_cfg.free_config = self.m_cfg.free_config
- plg_cfg.session_max_life = self.m_cfg.session_max_life
- plg_cfg.account.username = user_inputs.get("username", "")
- plg_cfg.account.password = user_inputs.get("password", "")
- if not plg_cfg.account.username:
- raise AccountIncompleteError(f"Task={task_id} missing username credentials in user_inputs")
-
- acceptable_keys = [target_routing_key]
- if self.m_cfg.need_proxy:
- proxy = ProxyManager.Instance().get_next_proxy(
- self.m_cfg.proxy_pool,
- self.m_cfg.proxy_cd,
- plugin_name=self.m_cfg.plugin_config.plugin_name
- )
- plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
- instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
- self._resources.track(instance)
- with self._resources.operation(instance):
- instance.set_log(self.m_logger)
- instance.set_config(plg_cfg)
- instance.create_session()
- task = Task(
- instance=instance,
- next_run=time.time(),
- task_ref=task_id,
- acceptable_routing_keys=acceptable_keys,
- source_queue=target_routing_key,
- book_allowed=True,
- next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
- )
- with self.m_lock:
- if self.m_stop_event.is_set():
- return
- self.m_tasks.append(task)
- success = True
- if plg_cfg.proxy.port > 0:
- ProxyManager.Instance().mark_proxy_result(
- proxy=plg_cfg.proxy.model_dump(),
- plugin_name=self.m_cfg.plugin_config.plugin_name,
- action="spawn",
- success=True,
- http_code=200
- )
- self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
- except Exception as e:
- self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
- finally:
- if not success and instance:
- self._cleanup_instance(instance, "spawn failed")
- if not success and task_id is not None:
- with self.m_lock:
- self.m_task_data_cache.pop(str(task_id), None)
- try:
- with self.m_lock:
- self._leased_orders.discard(str(task_id))
- # Retain a recovery deadline (including any 429 cooldown).
- self.redis_client.zadd(self.m_tracker_key, {
- str(task_id): max(time.time(), self.m_spawn_cooldown_until)})
- except Exception:
- pass
|