import os import time import json import random import threading from datetime import datetime, timedelta, timezone from typing import List, Dict, Callable, Optional from core_types import ( GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str, BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError, NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error ) from plugin_factory import PluginFactory from toolkit.cloud_api import CloudApi from toolkit.proxy_manager import ProxyManager from toolkit.notifier import notify_date_changes from utils.safe_redis_cli import SafeRedisClient class SentinelGCO: def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): self.m_cfg = cfg self.m_factory = PluginFactory() self.m_logger = logger self.m_tasks: List[Task] = [] self.m_lock = threading.RLock() self.m_stop_event = threading.Event() self.redis_client = SafeRedisClient(redis_conf, self.m_logger) self.m_last_group_query_time = 0.0 self.m_spawn_cooldown_until = 0.0 self.m_current_dates = {} def _log(self, message): if self.m_logger: self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}') else: print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}') def _get_average_interval(self) -> float: """Calculates the average query interval (in seconds) for the current group.""" mode = self.m_cfg.sentinel.query_wait.mode if mode == QueryWaitMode.Loop: return 1.0 elif mode == QueryWaitMode.Fixed: return float(self.m_cfg.sentinel.query_wait.fixed_wait) elif mode == QueryWaitMode.Random: return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0 return 30.0 def update_config(self, new_cfg: GroupConfig): """ Dynamically updates the configuration. """ with self.m_lock: if self.m_cfg.enable and not new_cfg.enable: self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.") elif not self.m_cfg.enable and new_cfg.enable: self._log("Config dynamically updated: Group ENABLED.") else: self._log("Config dynamically updated: Parameters refreshed.") self.m_cfg = new_cfg def get_status(self) -> GroupStatus: with self.m_lock: tasks = list(self.m_tasks) gid = self.m_cfg.identifier enable = self.m_cfg.enable plugin_name = self.m_cfg.plugin_config.plugin_name plugins_status = [] healthy_count = 0 for t in tasks: if t and t.instance: st = t.instance.get_plugin_status() if st.is_healthy: healthy_count += 1 plugins_status.append(st) return GroupStatus( group_id=gid, enable=enable, plugin_name=plugin_name, total_tasks=len(tasks), healthy_tasks=healthy_count, plugins=plugins_status, updated_at=utc_now_str(), ) def start(self): if not self.m_cfg.enable: return self._log("Starting Sentinel...") plugin_name = self.m_cfg.plugin_config.plugin_name class_name = "".join(part.title() for part in plugin_name.split('_')) plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin) self.m_factory.register_plugin(plugin_name, plugin_path, class_name) threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start() threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start() def stop(self): self._log("Stopping Sentinel...") self.m_stop_event.set() with self.m_lock: tasks_to_cleanup = list(self.m_tasks) self.m_tasks.clear() for task in tasks_to_cleanup: self._cleanup_task(task, "sentinel stopped") def _cleanup_instance(self, instance, reason: str = ""): try: if instance: self._log(f"๐Ÿงน Cleaned up instance. Reason: {reason}") instance.cleanup() except Exception as e: self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}") def _cleanup_task(self, task: Task, reason: str): if task and task.instance: self._cleanup_instance(task.instance, reason) def _remove_task(self, task: Task, reason: str): removed = False with self.m_lock: if task in self.m_tasks: self.m_tasks.remove(task) removed = True if removed: self._cleanup_task(task, reason) def _get_redis_key(self, routing_key: str) -> str: return f"vs:signal:{routing_key}" def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None: """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues.""" if acc and acc.id > 0 and acc.username not in ("", "Guest"): try: CloudApi.Instance().lock_account(acc.id, 0) self._log(f"๐Ÿ”“ [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).") except Exception as e: self._log(f"Failed to release account [{acc.username}]: {e}") def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None: """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data.""" if not acc or acc.id <= 0 or acc.username in ("", "Guest"): return try: cloud = CloudApi.Instance() url = f"{cloud.base_url}/api/account/update" params = {"account_id": acc.id} far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z") payload = { "status": "disable", "next_use_time": far_future, "extra_data": { "disable_reason": reason, "disabled_at": int(time.time()), "module": "SENTINEL", "group": getattr(self.m_cfg, "identifier", "") } } cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10) self._log(f"๐Ÿšซ [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}") except Exception as e: self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...") try: CloudApi.Instance().lock_account(acc.id, 86400 * 3650) except Exception as lock_err: self._log(f"Fallback 10-year lock also failed: {lock_err}") def _dispatch_error_policy( self, action: str, err: Exception, task: Optional[Task] = None, plg_cfg: Optional[PluginConfig] = None ) -> None: """ Sentinel error dispatch and self-healing policy engine: Executes self-healing actions based on the (action, error type) matrix: - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account, never releases back to pool. - 429 during spawn/login: Account-level rate limit. Locks account and sets m_spawn_cooldown_until backoff. - 429 during query: Session is valid; egress IP frequency limit. Preserves account, removes task to trigger creator spawning a new instance with a fresh proxy. - 401 / session expired: Removes broken task. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn. - Handles logging and timer backoffs internally. """ biz_err = normalize_error(err) active_plg_cfg = plg_cfg or (task.instance.config if task else None) acc = active_plg_cfg.account if active_plg_cfg else None # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found) if isinstance(biz_err, AccountStateError): self._log(f"๐Ÿšจ [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})") if acc and acc.id > 0 and acc.username not in ("", "Guest"): self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}") # Never release account back to pool if task: self._remove_task(task, f"account fatal error: {biz_err.message}") if action in ("spawn", "login", "create_session"): self.m_spawn_cooldown_until = time.time() + 5.0 # 1. Target site rate limit / 429 elif isinstance(biz_err, RateLimiteddError): if action in ("spawn", "login", "create_session"): backoff = float(self.m_cfg.login_backoff) self._log(f"๐Ÿ”’ [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).") if acc and acc.username not in ("", "Guest") and acc.id > 0: try: CloudApi.Instance().lock_account(acc.id, int(backoff)) self._log(f"๐Ÿ”’ Account [{acc.username}] locked for {int(backoff)}s via Cloud API.") except Exception as lock_err: self._log(f"Failed to lock account via cloud API: {lock_err}") self.m_spawn_cooldown_until = time.time() + backoff if task: self._remove_task(task, "spawn account rate limited") elif action == "query": # 429 during query: egress IP throttled, preserve account, remove current task to trigger auto-rotation to a new proxy self._log("๐Ÿ”„ [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...") if task: self._remove_task(task, "query ip rate limited") else: self._log(f"โš ๏ธ [Rate Limited] 429 encountered during {action}.") if task: task.next_run = time.time() + 10.0 # 2. Session expired / 401 elif isinstance(biz_err, SessionExpiredOrInvalidError): if action in ("spawn", "login", "create_session"): self._log(f"โŒ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...") self.m_spawn_cooldown_until = time.time() + 5.0 else: self._log(f"๐Ÿ”„ [Session Expired] Sentinel session expired during {action}. Removing task...") if task: self._remove_task(task, f"{action} session expired") # 3. Permission denied / WAF blocked / 403 elif isinstance(biz_err, PermissionDeniedError): proxy_cd = float(self.m_cfg.proxy_cd or 60.0) self._log(f"๐Ÿ›ก๏ธ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...") if task: self._remove_task(task, f"{action} waf blocked") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="WAF Blocked") self.m_spawn_cooldown_until = time.time() + proxy_cd # 4. Network or browser timeout elif isinstance(biz_err, NetworkTimeoutError): self._log(f"โฑ๏ธ [Timeout] Network/browser timeout during {action}.") if task: task.next_run = time.time() + 5.0 if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="Network Timeout") self.m_spawn_cooldown_until = time.time() + 10.0 # 5. Fallback policy else: self._log(f"โš ๏ธ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}") self.m_spawn_cooldown_until = time.time() + 10.0 elif task: task.next_run = time.time() + 10.0 # 6. Proxy health metrics recording if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=active_plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action=action, success=False, http_code=biz_err.http_status, error_type=biz_err.message ) def _is_within_active_hours(self) -> bool: """ Checks if current time is within allowed operating hours. """ start_str = self.m_cfg.active_time_start end_str = self.m_cfg.active_time_end current_bj_time = datetime.utcnow().time() start_time = datetime.strptime(start_str, "%H:%M").time() end_time = datetime.strptime(end_str, "%H:%M").time() return start_time <= current_bj_time <= end_time def _monitor_loop(self): self._log("Monitor loop started.") self.m_last_group_query_time = 0.0 while not self.m_stop_event.is_set(): try: time.sleep(1) now = time.time() with self.m_lock: tasks_to_check = list(self.m_tasks) active_tasks = [] dead_tasks = [] for t in tasks_to_check: if t.instance.health_check(): active_tasks.append(t) else: dead_tasks.append(t) if dead_tasks: with self.m_lock: current_tasks = list(self.m_tasks) self.m_tasks = [t for t in self.m_tasks if t in active_tasks] for t in dead_tasks: if t in current_tasks: self._cleanup_task(t, "health check failed") else: with self.m_lock: self.m_tasks = [t for t in self.m_tasks if t in active_tasks] if not active_tasks: continue avg_interval = self._get_average_interval() global_gap = max(1.0, avg_interval / len(active_tasks)) active_tasks.sort(key=lambda x: x.next_run) for task in active_tasks: if now < task.next_run: continue if now - self.m_last_group_query_time < global_gap: break apt_types = self.m_cfg.appointment_types if not apt_types: continue weights = [max(0.0, float(item.weight)) for item in apt_types] if sum(weights) > 0: apt_type = random.choices(apt_types, weights=weights, k=1)[0] else: apt_type = random.choice(apt_types) interval = 30 mode = task.qw_cfg.mode if mode == QueryWaitMode.Loop: interval = 1 elif mode == QueryWaitMode.Fixed: interval = task.qw_cfg.fixed_wait elif mode == QueryWaitMode.Random: interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max) self.m_last_group_query_time = now try: result = task.instance.query(apt_type) result.apt_type = apt_type notify_date_changes(result, cache=self.m_current_dates, logger=self._log) if result.success: ttl = self.m_cfg.sentinel.signal_ttl self._log(f"๐Ÿ”ฅ SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)") payload = { "group_id": self.m_cfg.identifier, "apt_type": apt_type.model_dump(), "query_result": result.to_snapshot_payload(), "timestamp": time.time() } redis_key = self._get_redis_key(apt_type.routing_key) self.redis_client.setex(redis_key, ttl, json.dumps(payload)) task.next_run = time.time() + interval # Record successful query metrics for proxy if task.instance.config.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=task.instance.config.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="query", success=True, http_code=200 ) except Exception as e: self._dispatch_error_policy("query", e, task=task) break except Exception as e: self._dispatch_error_policy("monitor_loop", e) def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue with self.m_lock: current = len(self.m_tasks) target = self.m_cfg.sentinel.target_instances if current < target: self._spawn_sentinel_worker() def _spawn_sentinel_worker(self): instance = None task = None success = False plg_cfg = None try: plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life if not self.m_cfg.need_account: plg_cfg.account.id = 0 plg_cfg.account.username = "Guest" else: acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd) plg_cfg.account = type(plg_cfg.account)(**acc) if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy( self.m_cfg.proxy_pool, self.m_cfg.proxy_cd, plugin_name=self.m_cfg.plugin_config.plugin_name ) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False) with self.m_lock: self.m_tasks.append(task) success = True if plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="spawn", success=True, http_code=200 ) self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed")