| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449 |
- import os
- from copy import deepcopy
- from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
- import time
- import random
- import threading
- from datetime import datetime, timedelta, timezone
- from typing import Dict, Callable, Optional
- from core_types import (
- BookerStandaloneConfig,
- QueryWaitMode,
- PluginConfig,
- Account,
- QueryResult,
- BookResult,
- GroupStatus,
- PluginStatus,
- utc_now_str,
- BizException,
- RateLimiteddError,
- SessionExpiredOrInvalidError,
- PermissionDeniedError,
- NetworkTimeoutError,
- AccountStateError,
- AccountAuthError,
- AccountBannedError,
- AccountIncompleteError,
- AccountNotFoundError,
- normalize_error
- )
- from plugin_factory import PluginFactory
- from toolkit.cloud_api import CloudApi
- from utils.safe_redis_cli import SafeRedisClient
- from toolkit.proxy_manager import ProxyManager
- from toolkit.notifier import notify_date_changes, notify_book_result
- class StandaloneGCO:
- """
- Standalone GCO Instance Manager.
- Adheres to the unified lifecycle contract of SentinelGCO / BookerGCO (start / stop / update_config).
- """
- TAG = "STANDALONE"
- def __init__(self, cfg: BookerStandaloneConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
- self.m_cfg = cfg
- self.m_logger = logger
- self.m_factory = PluginFactory()
- self.m_instance = None
- self.m_lock = threading.RLock()
- self.m_stop_event = threading.Event()
- self.m_last_login_time = 0
- self.m_next_query_time = 0
- self.m_next_keepalive_time = 0
- self.m_last_wait_log_time = 0.0
- self.m_current_dates = {}
- self._worker_thread = None
- self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
- self._booking_outcome = None
- init_lifecycle(self)
- def _log(self, message: str):
- prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
- if self.m_logger:
- self.m_logger(f"{prefix} {message}")
- else:
- print(f"{prefix} {message}")
- def _get_wait_interval(self) -> float:
- """Calculate wait interval for next query based on configuration."""
- mode = self.m_cfg.query_wait.mode
- if mode == QueryWaitMode.Loop:
- return 1.0
- elif mode == QueryWaitMode.Fixed:
- return max(1.0, float(self.m_cfg.query_wait.fixed_wait))
- elif mode == QueryWaitMode.Random:
- return max(1.0, random.uniform(self.m_cfg.query_wait.random_min, self.m_cfg.query_wait.random_max))
- return 30.0
- def _is_within_active_hours(self) -> bool:
- """Determine if current time is within active operating window for instances and queries (UTC)."""
- start_str = self.m_cfg.active_time_start
- end_str = self.m_cfg.active_time_end
- if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
- return True
- current_utc_time = datetime.now(timezone.utc).time()
- start_time = datetime.strptime(start_str, "%H:%M").time()
- end_time = datetime.strptime(end_str, "%H:%M").time()
- if start_time <= end_time:
- return start_time <= current_utc_time <= end_time
- else:
- return current_utc_time >= start_time or current_utc_time <= end_time
- def update_config(self, new_cfg: BookerStandaloneConfig):
- """Dynamically update group configuration."""
- with self.m_lock:
- if self.m_cfg.enable and not new_cfg.enable:
- self._log("Config dynamically updated: Group DISABLED.")
- elif not self.m_cfg.enable and new_cfg.enable:
- self._log("Config dynamically updated: Group ENABLED.")
- else:
- self._log("Config dynamically updated: Parameters refreshed.")
- self.m_cfg = new_cfg
- def get_status(self) -> GroupStatus:
- with self.m_lock:
- instance = self.m_instance
- gid = self.m_cfg.identifier
- enable = self.m_cfg.enable
- plugin_name = self.m_cfg.plugin_config.plugin_name
- plugins_status = []
- healthy_count = 0
- if instance:
- st = instance.get_plugin_status()
- if st.is_healthy:
- healthy_count += 1
- plugins_status.append(st)
- return GroupStatus(
- group_id=gid,
- enable=enable,
- plugin_name=plugin_name,
- total_tasks=1 if instance else 0,
- healthy_tasks=healthy_count,
- plugins=plugins_status,
- updated_at=utc_now_str(),
- )
- @start_once
- def start(self):
- """Non-blocking start: registers plugin and starts background worker thread."""
- if not self.m_cfg.enable:
- return
- self._log("Starting Standalone GCO...")
- plugin_name = self.m_cfg.plugin_config.plugin_name
- class_name = "".join(part.title() for part in plugin_name.split('_'))
- plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
- self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
- if plugin_name:
- self.m_factory.ensure_booking_supported(plugin_name)
- worker_name = f"Standalone-{self.m_cfg.identifier}"
- self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True, name=worker_name)
- self._worker_thread.start()
- @serialized_stop
- def stop(self):
- """Gracefully stop background worker and clean up resources."""
- self._log("Stopping Standalone GCO...")
- self.m_stop_event.set()
- threads = [self._worker_thread] if self._worker_thread else []
- join_threads(threads, self.m_cfg.shutdown_timeout)
- if self.m_instance is not None:
- self._resources.track(self.m_instance)
- self._resources.close(self.m_cfg.cleanup_timeout)
- threads_done = join_threads(threads, 0.1)
- if not self._resources.empty or not threads_done:
- raise RuntimeError('Standalone shutdown incomplete; resources retained for retry')
- self.m_instance = None
- self.redis_client.close()
- def _cleanup_instance(self, reason: str = ""):
- with self.m_lock:
- instance = self.m_instance
- if instance is None:
- return True
- cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
- if cleaned:
- with self.m_lock:
- if self.m_instance is instance:
- self.m_instance = None
- else:
- self._log(f"Cleanup retained for retry ({reason})")
- return cleaned
- def _init_instance(self):
- """Initialize single instance and log in to create a session."""
- if not self._resources.retry_failed():
- raise RuntimeError('Previous instance cleanup is still pending')
- if not self._cleanup_instance("recreating new session"):
- raise RuntimeError('Previous instance cleanup is still pending')
- now = time.time()
- if now < self.m_last_login_time + self.m_cfg.login_interval:
- wait_time = self.m_last_login_time + self.m_cfg.login_interval - now
- self._log(f"Waiting {int(wait_time)}s for login interval cooldown...")
- if self.m_stop_event.wait(wait_time):
- return
- plugin_name = self.m_cfg.plugin_config.plugin_name
- plg_cfg = PluginConfig()
- plg_cfg.debug = self.m_cfg.debug
- plg_cfg.account = self.m_cfg.account
- proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, proxy_cd=600)
- plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
- plg_cfg.free_config = self.m_cfg.free_config
- plg_cfg.session_max_life = self.m_cfg.session_max_life
- instance = self.m_factory.create(self.m_cfg.identifier, plugin_name)
- self._resources.track(instance)
- try:
- with self._resources.operation(instance):
- instance.set_log(self.m_logger)
- instance.set_config(plg_cfg)
- with self.m_lock:
- if self.m_stop_event.is_set():
- return
- self.m_instance = instance
- instance.create_session()
- except Exception:
- if self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout):
- with self.m_lock:
- if self.m_instance is instance:
- self.m_instance = None
- raise
- finally:
- if self.m_stop_event.is_set():
- self._resources.cleanup(instance, force=True, timeout=self.m_cfg.cleanup_timeout)
- self.m_last_login_time = time.time()
- interval = self._get_wait_interval()
- self.m_next_query_time = time.time() + interval
- self.m_next_keepalive_time = time.time() + 10.0
- self._log(f"Session created successfully. Initial query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')}).")
- def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
- """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
- if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
- return
- try:
- cloud = CloudApi.Instance()
- url = f"{cloud.base_url}/api/account/update"
- params = {"account_id": acc.id}
- far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
- payload = {
- "status": "disable",
- "next_use_time": far_future,
- "extra_data": {
- "disable_reason": reason,
- "disabled_at": int(time.time()),
- "module": "STANDALONE",
- "group": getattr(self.m_cfg, "identifier", "")
- }
- }
- cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
- self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
- except Exception as e:
- self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
- try:
- CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
- except Exception as lock_err:
- self._log(f"Fallback 10-year lock also failed: {lock_err}")
- def _dispatch_error_policy(self, action: str, err: Exception) -> None:
- biz_err = normalize_error(err)
- acc = self.m_cfg.account if (self.m_cfg.account and self.m_cfg.account.username not in ("", "Guest")) else None
- # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
- if isinstance(biz_err, AccountStateError):
- self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
- if acc and acc.id > 0 and acc.username not in ("", "Guest"):
- self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
- self._cleanup_instance(f"{action} account fatal error: {biz_err.message}")
- self.m_last_login_time = time.time()
- self.m_next_query_time = time.time() + 3600.0
- # 1. Target site rate limit / 429
- elif isinstance(biz_err, RateLimiteddError):
- if action in ("create_session", "login", "spawn"):
- backoff = float(self.m_cfg.login_interval)
- self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
- if acc and acc.id > 0:
- try:
- CloudApi.Instance().lock_account(acc.id, int(backoff))
- self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
- except Exception as lock_err:
- self._log(f"Failed to lock account via cloud API: {lock_err}")
- self.m_last_login_time = time.time()
- self._cleanup_instance(f"{action} account rate limited")
- elif action == "query":
- # 429 during query: Egress IP rate-limited, do not lock account, rotate proxy
- self._log("🔄 [Proxy Rate Limited] Query hit 429 (IP throttled, Account preserved). Cleaning instance to rotate proxy...")
- self._cleanup_instance("query ip rate limited")
- self.m_next_query_time = time.time() + 10.0
- else:
- self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
- self.m_next_query_time = time.time() + 10.0
- # 2. Session expired / 401
- elif isinstance(biz_err, SessionExpiredOrInvalidError):
- if action in ("create_session", "login", "spawn"):
- self._log(f"❌ [Auth Failed] Session creation failed (401 invalid credentials or challenge). Backing off for 300s...")
- self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0
- self._cleanup_instance(f"{action} auth failed")
- else:
- self._log(f"🔄 [Session Expired] Session died during {action}. Cleaning up instance for re-login in 5s...")
- self._cleanup_instance(f"{action} session expired")
- self.m_next_query_time = time.time() + 5.0
- # 3. Permission denied / WAF blocked / 403
- elif isinstance(biz_err, PermissionDeniedError):
- self._cleanup_instance(f"{action} waf blocked")
- if action in ("create_session", "login", "spawn"):
- self._log(f"🛡️ [WAF Blocked] 403 Forbidden detected during {action}. Cooling down for 300s...")
- self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0
- else:
- self._log(f"🛡️ [WAF Blocked] 403 Forbidden detected during {action}. Cleaning instance to rotate proxy (60s cooldown)...")
- self.m_next_query_time = time.time() + 60.0
- # 4. Network or browser timeout
- elif isinstance(biz_err, NetworkTimeoutError):
- if action in ("create_session", "login", "spawn"):
- self._log(f"⏱️ [Timeout] Network/browser timeout during {action}. Cooling down for 300s...")
- self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0
- self._cleanup_instance(f"{action} timeout")
- else:
- self._log(f"⏱️ [Timeout] Network/browser timeout during {action}. Retrying in 5s...")
- self.m_next_query_time = time.time() + 5.0
- # 5. Fallback policy
- else:
- self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
- if action in ("create_session", "worker_loop"):
- self._log(f"🛑 [Session Penalty] {action} failed. Cooling down for 300s (5 min) to prevent traffic burn...")
- self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0
- self._cleanup_instance(f"{action} unexpected error")
- else:
- self.m_next_query_time = time.time() + 10.0
- def _worker_loop(self):
- """
- Background worker daemon loop: keepalive -> schedule check -> query -> booking
- """
- self._log("Standalone worker loop started.")
- while not self.m_stop_event.is_set():
- try:
- key = (self.m_cfg.identifier, self.m_cfg.account.username,
- self.m_cfg.user_preferences.get('booking_run_id', 'default'))
- event_id = 'standalone:' + repr(key)
- if self._booking_outcome and self._booking_outcome[0] == key:
- receipt = self._booking_outcome[1]
- notify_book_result(receipt, event_id='booking:' + event_id, logger=self._log)
- self._cleanup_instance('booking completed')
- self.m_stop_event.wait(5)
- continue
- if not self.m_cfg.enable:
- self.m_stop_event.wait(1.0)
- continue
- if not self._is_within_active_hours():
- self.m_stop_event.wait(5.0)
- continue
- # 1. Session initialization and self-healing
- if self.m_instance is None:
- try:
- self._init_instance()
- if self.m_stop_event.is_set():
- break
- except Exception as e:
- self._dispatch_error_policy("create_session", e)
- continue
- # 2. Health check
- if self.m_instance and not self.m_instance.health_check():
- self._log("Health check failed. Session dead. Recreating session...")
- self._cleanup_instance("session health check failed")
- continue
- # 3. Query interval wait and periodic keepalive
- now = time.time()
- if now < self.m_next_query_time:
- remaining = self.m_next_query_time - now
- next_query_str = datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')
- keep_alive_sec = self.m_cfg.keep_alive
- # Periodic keepalive
- if self.m_instance and now >= self.m_next_keepalive_time:
- with self._resources.operation(self.m_instance):
- self.m_instance.keep_alive()
- self.m_next_keepalive_time = now + keep_alive_sec
- self._log(f"Session keepalive sent. Next query at {next_query_str} (remaining: {remaining:.0f}s)...")
- self.m_last_wait_log_time = now
- # Periodic wait countdown (logged every 15s)
- elif now - self.m_last_wait_log_time >= 15.0 and remaining > 3.0:
- self.m_last_wait_log_time = now
- self._log(f"Waiting for next query at {next_query_str} (remaining: {remaining:.0f}s)...")
- wait_sec = min(remaining, 1.0)
- self.m_stop_event.wait(wait_sec)
- continue
- # 4. Randomly pick a routing_key for query
- apt_types = self.m_cfg.appointment_types
- if not apt_types:
- self.m_stop_event.wait(2.0)
- continue
- weights = [max(0.0, float(t.weight)) for t in apt_types]
- if sum(weights) > 0:
- apt_type = random.choices(apt_types, weights=weights, k=1)[0]
- else:
- apt_type = random.choice(apt_types)
- self._log(f"Querying slots for {apt_type.routing_key}...")
- try:
- with self._resources.operation(self.m_instance):
- query_result = self.m_instance.query(apt_type)
- query_result.apt_type = apt_type
- notify_date_changes(query_result, cache=self.m_current_dates, logger=self._log)
- except Exception as e:
- self._dispatch_error_policy("query", e)
- continue
- # 5. Slot found: trigger auto-booking
- if query_result.success:
- self._log("🔥 SLOT FOUND! Initiating AUTO-BOOKING...")
- try:
- if self.m_stop_event.is_set():
- break
- with self._resources.operation(self.m_instance):
- book_result = self.m_instance.book(query_result, deepcopy(self.m_cfg.user_preferences))
- notify_book_result(book_result, logger=self._log, event_id='booking:' + event_id)
- if book_result.success:
- self._booking_outcome = (key, book_result)
- self._log(f"🎉 BOOKING SUCCESSFUL for {apt_type.routing_key}!")
- self.m_stop_event.wait(5.0)
- except Exception as e:
- self._dispatch_error_policy("book", e)
- # 6. Recompute and schedule next query and keepalive times
- interval = self._get_wait_interval()
- now = time.time()
- self.m_next_query_time = now + interval
- keep_alive_sec = self.m_cfg.keep_alive
- self.m_next_keepalive_time = now + keep_alive_sec
- self.m_last_wait_log_time = now
- self._log(f"Next query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')})...")
- except Exception as e:
- self._dispatch_error_policy("worker_loop", e)
|