import os from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads import time import json import random import threading from datetime import datetime, timedelta, timezone from typing import List, Dict, Callable, Optional from core_types import ( GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str, BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError, NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError, AccountIncompleteError, AccountNotFoundError, normalize_error ) from plugin_factory import PluginFactory from toolkit.cloud_api import CloudApi from toolkit.proxy_manager import ProxyManager from toolkit.notifier import notify_date_changes from utils.safe_redis_cli import SafeRedisClient from toolkit.task_executor import TaskExecutor class SentinelGCO: def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None): self.m_cfg = cfg self.m_factory = PluginFactory() self.m_logger = logger self.m_tasks: List[Task] = [] self.m_lock = threading.RLock() self.m_stop_event = threading.Event() self.redis_client = SafeRedisClient(redis_conf, self.m_logger) init_lifecycle(self) self.m_last_group_query_time = 0.0 self.m_spawn_cooldown_until = 0.0 self.m_current_dates = {} self._executor = TaskExecutor(cfg.sentinel.max_concurrency, 'Sentinel') self._background_threads = [] def _log(self, message): if self.m_logger: self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}') else: print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}') def _get_average_interval(self) -> float: """Calculates the average query interval (in seconds) for the current group.""" mode = self.m_cfg.sentinel.query_wait.mode if mode == QueryWaitMode.Loop: return 1.0 elif mode == QueryWaitMode.Fixed: return float(self.m_cfg.sentinel.query_wait.fixed_wait) elif mode == QueryWaitMode.Random: return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0 return 30.0 def update_config(self, new_cfg: GroupConfig): """ Dynamically updates the configuration. """ with self.m_lock: if self.m_cfg.enable and not new_cfg.enable: self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.") elif not self.m_cfg.enable and new_cfg.enable: self._log("Config dynamically updated: Group ENABLED.") else: self._log("Config dynamically updated: Parameters refreshed.") self.m_cfg = new_cfg def get_status(self) -> GroupStatus: with self.m_lock: tasks = list(self.m_tasks) gid = self.m_cfg.identifier enable = self.m_cfg.enable plugin_name = self.m_cfg.plugin_config.plugin_name plugins_status = [] healthy_count = 0 for t in tasks: if t and t.instance: st = t.instance.get_plugin_status() if st.is_healthy: healthy_count += 1 plugins_status.append(st) return GroupStatus( group_id=gid, enable=enable, plugin_name=plugin_name, total_tasks=len(tasks), healthy_tasks=healthy_count, plugins=plugins_status, updated_at=utc_now_str(), ) @start_once def start(self): if not self.m_cfg.enable: return self._log("Starting Sentinel...") plugin_name = self.m_cfg.plugin_config.plugin_name class_name = "".join(part.title() for part in plugin_name.split('_')) plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin) self.m_factory.register_plugin(plugin_name, plugin_path, class_name) for target in (self._monitor_loop, self._creator_loop): thread = threading.Thread(target=target, daemon=True) self._background_threads.append(thread) thread.start() @serialized_stop def stop(self): self._log("Stopping Sentinel...") self.m_stop_event.set() deadline = time.monotonic() + self.m_cfg.shutdown_timeout self._executor.shutdown(timeout=max(0, deadline - time.monotonic())) join_threads(self._background_threads, max(0, deadline - time.monotonic())) with self.m_lock: for task in self.m_tasks: task.removed = True self._resources.track(task.instance) self._resources.close(self.m_cfg.cleanup_timeout) threads_done = self._executor.shutdown(timeout=0.1) and join_threads(self._background_threads, 0.1) if not self._resources.empty or not threads_done: raise RuntimeError('Sentinel shutdown incomplete; resources retained for retry') with self.m_lock: self.m_tasks.clear() self.redis_client.close() def _cleanup_instance(self, instance, reason: str = ""): if instance is None: return True cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout) if not cleaned: self._log(f"Cleanup retained for retry ({reason})") return cleaned def _cleanup_task(self, task: Task, reason: str): if task and task.instance: with task._operation_lock: self._cleanup_instance(task.instance, reason) def _remove_task(self, task: Task, reason: str): removed = False with self.m_lock: if task in self.m_tasks: task.removed = True self.m_tasks.remove(task) removed = True if removed: self._cleanup_task(task, reason) def _get_redis_key(self, routing_key: str) -> str: return f"vs:signal:{routing_key}" def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None: """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues.""" if acc and acc.id > 0 and acc.username not in ("", "Guest"): try: CloudApi.Instance().lock_account(acc.id, 0) self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).") except Exception as e: self._log(f"Failed to release account [{acc.username}]: {e}") def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None: """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data.""" if not acc or acc.id <= 0 or acc.username in ("", "Guest"): return try: cloud = CloudApi.Instance() url = f"{cloud.base_url}/api/account/update" params = {"account_id": acc.id} far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z") payload = { "status": "disable", "next_use_time": far_future, "extra_data": { "disable_reason": reason, "disabled_at": int(time.time()), "module": "SENTINEL", "group": getattr(self.m_cfg, "identifier", "") } } cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10) self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}") except Exception as e: self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...") try: CloudApi.Instance().lock_account(acc.id, 86400 * 3650) except Exception as lock_err: self._log(f"Fallback 10-year lock also failed: {lock_err}") def _dispatch_error_policy( self, action: str, err: Exception, task: Optional[Task] = None, plg_cfg: Optional[PluginConfig] = None ) -> None: """ Sentinel error dispatch and self-healing policy engine: Executes self-healing actions based on the (action, error type) matrix: - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account, never releases back to pool. - 429 during spawn/login: Account-level rate limit. Locks account and sets m_spawn_cooldown_until backoff. - 429 during query: Session is valid; egress IP frequency limit. Preserves account, removes task to trigger creator spawning a new instance with a fresh proxy. - 401 / session expired: Removes broken task. - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn. - Handles logging and timer backoffs internally. """ biz_err = normalize_error(err) active_plg_cfg = plg_cfg or (task.instance.config if task else None) acc = active_plg_cfg.account if active_plg_cfg else None # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found) if isinstance(biz_err, AccountStateError): self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})") if acc and acc.id > 0 and acc.username not in ("", "Guest"): self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}") # Never release account back to pool if task: self._remove_task(task, f"account fatal error: {biz_err.message}") if action in ("spawn", "login", "create_session"): self.m_spawn_cooldown_until = time.time() + 5.0 # 1. Target site rate limit / 429 elif isinstance(biz_err, RateLimiteddError): if action in ("spawn", "login", "create_session"): backoff = float(self.m_cfg.login_backoff) self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).") if acc and acc.username not in ("", "Guest") and acc.id > 0: try: CloudApi.Instance().lock_account(acc.id, int(backoff)) self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.") except Exception as lock_err: self._log(f"Failed to lock account via cloud API: {lock_err}") self.m_spawn_cooldown_until = time.time() + backoff if task: self._remove_task(task, "spawn account rate limited") elif action == "query": # 429 during query: egress IP throttled, preserve account, remove current task to trigger auto-rotation to a new proxy self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...") if task: self._remove_task(task, "query ip rate limited") else: self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.") if task: task.next_run = time.time() + 10.0 # 2. Session expired / 401 elif isinstance(biz_err, SessionExpiredOrInvalidError): if action in ("spawn", "login", "create_session"): self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...") self.m_spawn_cooldown_until = time.time() + 5.0 else: self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...") if task: self._remove_task(task, f"{action} session expired") # 3. Permission denied / WAF blocked / 403 elif isinstance(biz_err, PermissionDeniedError): proxy_cd = float(self.m_cfg.proxy_cd or 60.0) self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...") if task: self._remove_task(task, f"{action} waf blocked") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="WAF Blocked") self.m_spawn_cooldown_until = time.time() + proxy_cd # 4. Network or browser timeout elif isinstance(biz_err, NetworkTimeoutError): self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.") if task: task.next_run = time.time() + 5.0 if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason="Network Timeout") self.m_spawn_cooldown_until = time.time() + 10.0 # 5. Fallback policy else: self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})") if action in ("spawn", "login", "create_session"): self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}") self.m_spawn_cooldown_until = time.time() + 10.0 elif task: task.next_run = time.time() + 10.0 # 6. Proxy health metrics recording if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=active_plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action=action, success=False, http_code=biz_err.http_status, error_type=biz_err.message ) def _is_within_active_hours(self) -> bool: """ Checks if current time is within allowed operating hours (UTC). """ start_str = self.m_cfg.active_time_start end_str = self.m_cfg.active_time_end if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str: return True current_utc_time = datetime.now(timezone.utc).time() start_time = datetime.strptime(start_str, "%H:%M").time() end_time = datetime.strptime(end_str, "%H:%M").time() if start_time <= end_time: return start_time <= current_utc_time <= end_time else: return current_utc_time >= start_time or current_utc_time <= end_time def _monitor_task(self, task, apt_type): try: with task._operation_lock, self._resources.operation(task.instance): if self.m_stop_event.is_set() or task.removed: return now = time.time() if now >= task.next_health_check: if not task.instance.health_check(): self._remove_task(task, "health check failed") return task.next_health_check = now + 10 if apt_type is None: return qw = self.m_cfg.sentinel.query_wait if qw.mode == QueryWaitMode.Loop: interval = 1 elif qw.mode == QueryWaitMode.Fixed: interval = qw.fixed_wait else: interval = random.uniform(qw.random_min, qw.random_max) result = task.instance.query(apt_type) result.apt_type = apt_type with self.m_lock: notify_date_changes(result, cache=self.m_current_dates, logger=self._log) if result.success: payload = {"group_id": self.m_cfg.identifier, "apt_type": apt_type.model_dump(), "query_result": result.to_snapshot_payload(), "timestamp": time.time()} self.redis_client.setex(self._get_redis_key(apt_type.routing_key), self.m_cfg.sentinel.signal_ttl, json.dumps(payload)) task.next_run = time.time() + max(0.05, interval) if task.instance.config.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=task.instance.config.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="query", success=True, http_code=200) except Exception as exc: self._dispatch_error_policy("query", exc, task=task) finally: with self.m_lock: task.busy = False def _schedule_monitor_jobs(self): with self.m_lock: tasks = sorted(self.m_tasks, key=lambda t: t.next_run) gap = max(0.05, self._get_average_interval() / max(1, len(tasks))) for task in tasks: if task.busy or task.removed or self.m_stop_event.is_set(): continue now = time.time() apt_type = None apt_types = self.m_cfg.appointment_types if (apt_types and now >= task.next_run and now - self.m_last_group_query_time >= gap): weights = [max(0, apt.weight) for apt in apt_types] apt_type = (random.choices(apt_types, weights=weights, k=1)[0] if sum(weights) else random.choice(apt_types)) if apt_type is None and now < task.next_health_check: continue task.busy = True if self._executor.submit(self._monitor_task, task, apt_type): if apt_type is not None: self.m_last_group_query_time = now else: task.busy = False def _monitor_loop(self): self._log("Monitor loop started.") while not self.m_stop_event.wait(0.05): try: if self.m_cfg.enable and self._is_within_active_hours(): self._schedule_monitor_jobs() except Exception as exc: self._dispatch_error_policy("monitor_loop", exc) def _creator_loop(self): self._log("Creator loop started.") while not self.m_stop_event.wait(1.0): if not self._is_within_active_hours(): continue if time.time() < self.m_spawn_cooldown_until: continue with self.m_lock: current = len(self.m_tasks) target = self.m_cfg.sentinel.target_instances if current < target: self._spawn_sentinel_worker() def _spawn_sentinel_worker(self): if self.m_stop_event.is_set() or not self._resources.retry_failed(): return instance = None task = None success = False plg_cfg = None try: plg_cfg = PluginConfig() plg_cfg.debug = self.m_cfg.debug plg_cfg.free_config = self.m_cfg.free_config plg_cfg.session_max_life = self.m_cfg.session_max_life if not self.m_cfg.need_account: plg_cfg.account.id = 0 plg_cfg.account.username = "Guest" else: acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd) plg_cfg.account = type(plg_cfg.account)(**acc) if self.m_cfg.need_proxy: proxy = ProxyManager.Instance().get_next_proxy( self.m_cfg.proxy_pool, self.m_cfg.proxy_cd, plugin_name=self.m_cfg.plugin_config.plugin_name ) plg_cfg.proxy = type(plg_cfg.proxy)(**proxy) instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name) self._resources.track(instance) with self._resources.operation(instance): instance.set_log(self.m_logger) instance.set_config(plg_cfg) instance.create_session() task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False) with self.m_lock: if self.m_stop_event.is_set(): return self.m_tasks.append(task) success = True if plg_cfg.proxy.port > 0: ProxyManager.Instance().mark_proxy_result( proxy=plg_cfg.proxy.model_dump(), plugin_name=self.m_cfg.plugin_config.plugin_name, action="spawn", success=True, http_code=200 ) self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}") except Exception as e: self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg) finally: if not success and instance: self._cleanup_instance(instance, "spawn failed")