import os from copy import deepcopy from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads import time import random import threading from datetime import datetime, timedelta, timezone from typing import Dict, Callable, Optional from core_types import ( BookerStandaloneConfig, QueryWaitMode, PluginConfig, Account, QueryResult, BookResult, GroupStatus, PluginStatus, utc_now_str, BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError, NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error ) from plugin_factory import PluginFactory from toolkit.cloud_api import CloudApi from utils.safe_redis_cli import SafeRedisClient from toolkit.proxy_manager import ProxyManager from toolkit.notifier import notify_date_changes, notify_book_result class StandaloneGCO: """ Standalone GCO Instance Manager. Adheres to the unified lifecycle contract of SentinelGCO / BookerGCO (start / stop / update_config). """ TAG = "STANDALONE" def __init__(self, cfg: BookerStandaloneConfig, redis_conf: Dict, logger: Callable[[str], None] = None): self.m_cfg = cfg self.m_logger = logger self.m_factory = PluginFactory() self.m_instance = None self.m_lock = threading.RLock() self.m_stop_event = threading.Event() self.m_last_login_time = 0 self.m_next_query_time = 0 self.m_next_keepalive_time = 0 self.m_last_wait_log_time = 0.0 self.m_current_dates = {} self._worker_thread = None self.redis_client = SafeRedisClient(redis_conf, self.m_logger) self._booking_outcome = None init_lifecycle(self) def _log(self, message: str): prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]" if self.m_logger: self.m_logger(f"{prefix} {message}") else: print(f"{prefix} {message}") def _get_wait_interval(self) -> float: """Calculate wait interval for next query based on configuration.""" mode = self.m_cfg.query_wait.mode if mode == QueryWaitMode.Loop: return 1.0 elif mode == QueryWaitMode.Fixed: return max(1.0, float(self.m_cfg.query_wait.fixed_wait)) elif mode == QueryWaitMode.Random: return max(1.0, random.uniform(self.m_cfg.query_wait.random_min, self.m_cfg.query_wait.random_max)) return 30.0 def _is_within_active_hours(self) -> bool: """Determine if current time is within active operating window for instances and queries (UTC).""" start_str = self.m_cfg.active_time_start end_str = self.m_cfg.active_time_end if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str: return True current_utc_time = datetime.now(timezone.utc).time() start_time = datetime.strptime(start_str, "%H:%M").time() end_time = datetime.strptime(end_str, "%H:%M").time() if start_time <= end_time: return start_time <= current_utc_time <= end_time else: return current_utc_time >= start_time or current_utc_time <= end_time def update_config(self, new_cfg: BookerStandaloneConfig): """Dynamically update group configuration.""" with self.m_lock: if self.m_cfg.enable and not new_cfg.enable: self._log("Config dynamically updated: Group DISABLED.") elif not self.m_cfg.enable and new_cfg.enable: self._log("Config dynamically updated: Group ENABLED.") else: self._log("Config dynamically updated: Parameters refreshed.") self.m_cfg = new_cfg def get_status(self) -> GroupStatus: with self.m_lock: instance = self.m_instance gid = self.m_cfg.identifier enable = self.m_cfg.enable plugin_name = self.m_cfg.plugin_config.plugin_name plugins_status = [] healthy_count = 0 if instance: st = instance.get_plugin_status() if st.is_healthy: healthy_count += 1 plugins_status.append(st) return GroupStatus( group_id=gid, enable=enable, plugin_name=plugin_name, total_tasks=1 if instance else 0, healthy_tasks=healthy_count, plugins=plugins_status, updated_at=utc_now_str(), ) @start_once def start(self): """Non-blocking start: registers plugin and starts background worker thread.""" if not self.m_cfg.enable: return self._log("Starting Standalone GCO...") plugin_name = self.m_cfg.plugin_config.plugin_name class_name = "".join(part.title() for part in plugin_name.split('_')) plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin) self.m_factory.register_plugin(plugin_name, plugin_path, class_name) if plugin_name: self.m_factory.ensure_booking_supported(plugin_name) worker_name = f"Standalone-{self.m_cfg.identifier}" self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True, name=worker_name) self._worker_thread.start() @serialized_stop def stop(self): """Gracefully stop background worker and clean up resources.""" self._log("Stopping Standalone GCO...") self.m_stop_event.set() threads = [self._worker_thread] if self._worker_thread else [] join_threads(threads, self.m_cfg.shutdown_timeout) if self.m_instance is not None: self._resources.track(self.m_instance) self._resources.close(self.m_cfg.cleanup_timeout) threads_done = join_threads(threads, 0.1) if not self._resources.empty or not threads_done: raise RuntimeError('Standalone shutdown incomplete; resources retained for retry') self.m_instance = None self.redis_client.close() def _cleanup_instance(self, reason: str = ""): with self.m_lock: instance = self.m_instance if instance is None: return True cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout) if cleaned: with self.m_lock: if self.m_instance is instance: self.m_instance = None else: self._log(f"Cleanup retained for retry ({reason})") return cleaned def _init_instance(self): """Initialize single instance and log in to create a session.""" if not self._resources.retry_failed(): raise RuntimeError('Previous instance cleanup is still pending') if not self._cleanup_instance("recreating new session"): raise RuntimeError('Previous instance cleanup is still pending') now = time.time() if now < self.m_last_login_time + self.m_cfg.login_interval: wait_time = self.m_last_login_time + self.m_cfg.login_interval - now self._log(f"Waiting {int(wait_time)}s for login interval cooldown...") if self.m_stop_event.wait(wait_time): return plugin_name = self.m_cfg.plugin_config.plugin_name plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.account = self.m_cfg.account proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, proxy_cd=600) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life instance = self.m_factory.create(self.m_cfg.identifier, plugin_name) self._resources.track(instance) try: with self._resources.operation(instance): instance.set_log(self.m_logger) instance.set_config(plg_cfg) with self.m_lock: if self.m_stop_event.is_set(): return self.m_instance = instance instance.create_session() except Exception: if self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout): with self.m_lock: if self.m_instance is instance: self.m_instance = None raise finally: if self.m_stop_event.is_set(): self._resources.cleanup(instance, force=True, timeout=self.m_cfg.cleanup_timeout) self.m_last_login_time = time.time() interval = self._get_wait_interval() self.m_next_query_time = time.time() + interval self.m_next_keepalive_time = time.time() + 10.0 self._log(f"Session created successfully. Initial query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')}).") def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None: """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data.""" if not acc or acc.id <= 0 or acc.username in ("", "Guest"): return try: cloud = CloudApi.Instance() url = f"{cloud.base_url}/api/account/update" params = {"account_id": acc.id} far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z") payload = { "status": "disable", "next_use_time": far_future, "extra_data": { "disable_reason": reason, "disabled_at": int(time.time()), "module": "STANDALONE", "group": getattr(self.m_cfg, "identifier", "") } } cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10) self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}") except Exception as e: self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...") try: CloudApi.Instance().lock_account(acc.id, 86400 * 3650) except Exception as lock_err: self._log(f"Fallback 10-year lock also failed: {lock_err}") def _dispatch_error_policy(self, action: str, err: Exception) -> None: biz_err = normalize_error(err) acc = self.m_cfg.account if (self.m_cfg.account and self.m_cfg.account.username not in ("", "Guest")) else None # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found) if isinstance(biz_err, AccountStateError): self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})") if acc and acc.id > 0 and acc.username not in ("", "Guest"): self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}") self._cleanup_instance(f"{action} account fatal error: {biz_err.message}") self.m_last_login_time = time.time() self.m_next_query_time = time.time() + 3600.0 # 1. Target site rate limit / 429 elif isinstance(biz_err, RateLimiteddError): if action in ("create_session", "login", "spawn"): backoff = float(self.m_cfg.login_interval) self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).") if acc and acc.id > 0: try: CloudApi.Instance().lock_account(acc.id, int(backoff)) self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.") except Exception as lock_err: self._log(f"Failed to lock account via cloud API: {lock_err}") self.m_last_login_time = time.time() self._cleanup_instance(f"{action} account rate limited") elif action == "query": # 429 during query: Egress IP rate-limited, do not lock account, rotate proxy self._log("🔄 [Proxy Rate Limited] Query hit 429 (IP throttled, Account preserved). Cleaning instance to rotate proxy...") self._cleanup_instance("query ip rate limited") self.m_next_query_time = time.time() + 10.0 else: self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.") self.m_next_query_time = time.time() + 10.0 # 2. Session expired / 401 elif isinstance(biz_err, SessionExpiredOrInvalidError): if action in ("create_session", "login", "spawn"): self._log(f"❌ [Auth Failed] Session creation failed (401 invalid credentials or challenge). Backing off for 300s...") self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0 self._cleanup_instance(f"{action} auth failed") else: self._log(f"🔄 [Session Expired] Session died during {action}. Cleaning up instance for re-login in 5s...") self._cleanup_instance(f"{action} session expired") self.m_next_query_time = time.time() + 5.0 # 3. Permission denied / WAF blocked / 403 elif isinstance(biz_err, PermissionDeniedError): self._cleanup_instance(f"{action} waf blocked") if action in ("create_session", "login", "spawn"): self._log(f"🛡️ [WAF Blocked] 403 Forbidden detected during {action}. Cooling down for 300s...") self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0 else: self._log(f"🛡️ [WAF Blocked] 403 Forbidden detected during {action}. Cleaning instance to rotate proxy (60s cooldown)...") self.m_next_query_time = time.time() + 60.0 # 4. Network or browser timeout elif isinstance(biz_err, NetworkTimeoutError): if action in ("create_session", "login", "spawn"): self._log(f"⏱️ [Timeout] Network/browser timeout during {action}. Cooling down for 300s...") self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0 self._cleanup_instance(f"{action} timeout") else: self._log(f"⏱️ [Timeout] Network/browser timeout during {action}. Retrying in 5s...") self.m_next_query_time = time.time() + 5.0 # 5. Fallback policy else: self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})") if action in ("create_session", "worker_loop"): self._log(f"🛑 [Session Penalty] {action} failed. Cooling down for 300s (5 min) to prevent traffic burn...") self.m_last_login_time = time.time() - self.m_cfg.login_interval + 300.0 self._cleanup_instance(f"{action} unexpected error") else: self.m_next_query_time = time.time() + 10.0 def _worker_loop(self): """ Background worker daemon loop: keepalive -> schedule check -> query -> booking """ self._log("Standalone worker loop started.") while not self.m_stop_event.is_set(): try: key = (self.m_cfg.identifier, self.m_cfg.account.username, self.m_cfg.user_preferences.get('booking_run_id', 'default')) event_id = 'standalone:' + repr(key) if self._booking_outcome and self._booking_outcome[0] == key: receipt = self._booking_outcome[1] notify_book_result(receipt, event_id='booking:' + event_id, logger=self._log) self._cleanup_instance('booking completed') self.m_stop_event.wait(5) continue if not self.m_cfg.enable: self.m_stop_event.wait(1.0) continue if not self._is_within_active_hours(): self.m_stop_event.wait(5.0) continue # 1. Session initialization and self-healing if self.m_instance is None: try: self._init_instance() if self.m_stop_event.is_set(): break except Exception as e: self._dispatch_error_policy("create_session", e) continue # 2. Health check if self.m_instance and not self.m_instance.health_check(): self._log("Health check failed. Session dead. Recreating session...") self._cleanup_instance("session health check failed") continue # 3. Query interval wait and periodic keepalive now = time.time() if now < self.m_next_query_time: remaining = self.m_next_query_time - now next_query_str = datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S') keep_alive_sec = self.m_cfg.keep_alive # Periodic keepalive if self.m_instance and now >= self.m_next_keepalive_time: with self._resources.operation(self.m_instance): self.m_instance.keep_alive() self.m_next_keepalive_time = now + keep_alive_sec self._log(f"Session keepalive sent. Next query at {next_query_str} (remaining: {remaining:.0f}s)...") self.m_last_wait_log_time = now # Periodic wait countdown (logged every 15s) elif now - self.m_last_wait_log_time >= 15.0 and remaining > 3.0: self.m_last_wait_log_time = now self._log(f"Waiting for next query at {next_query_str} (remaining: {remaining:.0f}s)...") wait_sec = min(remaining, 1.0) self.m_stop_event.wait(wait_sec) continue # 4. Randomly pick a routing_key for query apt_types = self.m_cfg.appointment_types if not apt_types: self.m_stop_event.wait(2.0) continue weights = [max(0.0, float(t.weight)) for t in apt_types] if sum(weights) > 0: apt_type = random.choices(apt_types, weights=weights, k=1)[0] else: apt_type = random.choice(apt_types) self._log(f"Querying slots for {apt_type.routing_key}...") try: with self._resources.operation(self.m_instance): query_result = self.m_instance.query(apt_type) query_result.apt_type = apt_type notify_date_changes(query_result, cache=self.m_current_dates, logger=self._log) except Exception as e: self._dispatch_error_policy("query", e) continue # 5. Slot found: trigger auto-booking if query_result.success: self._log("🔥 SLOT FOUND! Initiating AUTO-BOOKING...") try: if self.m_stop_event.is_set(): break with self._resources.operation(self.m_instance): book_result = self.m_instance.book(query_result, deepcopy(self.m_cfg.user_preferences)) notify_book_result(book_result, logger=self._log, event_id='booking:' + event_id) if book_result.success: self._booking_outcome = (key, book_result) self._log(f"🎉 BOOKING SUCCESSFUL for {apt_type.routing_key}!") self.m_stop_event.wait(5.0) except Exception as e: self._dispatch_error_policy("book", e) # 6. Recompute and schedule next query and keepalive times interval = self._get_wait_interval() now = time.time() self.m_next_query_time = now + interval keep_alive_sec = self.m_cfg.keep_alive self.m_next_keepalive_time = now + keep_alive_sec self.m_last_wait_log_time = now self._log(f"Next query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')})...") except Exception as e: self._dispatch_error_policy("worker_loop", e)