| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464 |
- import os
- from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
- import time
- import json
- import random
- import threading
- from datetime import datetime, timedelta, timezone
- from typing import List, Dict, Callable, Optional
- from core_types import (
- GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
- BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
- NetworkTimeoutError, AccountStateError, AccountAuthError, AccountBannedError,
- AccountIncompleteError, AccountNotFoundError, normalize_error
- )
- from plugin_factory import PluginFactory
- from toolkit.cloud_api import CloudApi
- from toolkit.proxy_manager import ProxyManager
- from toolkit.notifier import notify_date_changes
- from utils.safe_redis_cli import SafeRedisClient
- from toolkit.task_executor import TaskExecutor
- class SentinelGCO:
- def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
- self.m_cfg = cfg
- self.m_factory = PluginFactory()
- self.m_logger = logger
- self.m_tasks: List[Task] = []
- self.m_lock = threading.RLock()
- self.m_stop_event = threading.Event()
- self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
- init_lifecycle(self)
- self.m_last_group_query_time = 0.0
- self.m_spawn_cooldown_until = 0.0
- self.m_current_dates = {}
- self._executor = TaskExecutor(cfg.sentinel.max_concurrency, 'Sentinel')
- self._background_threads = []
- def _log(self, message):
- if self.m_logger:
- self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
- else:
- print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
-
- def _get_average_interval(self) -> float:
- """Calculates the average query interval (in seconds) for the current group."""
- mode = self.m_cfg.sentinel.query_wait.mode
- if mode == QueryWaitMode.Loop:
- return 1.0
- elif mode == QueryWaitMode.Fixed:
- return float(self.m_cfg.sentinel.query_wait.fixed_wait)
- elif mode == QueryWaitMode.Random:
- return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
- return 30.0
-
- def update_config(self, new_cfg: GroupConfig):
- """
- Dynamically updates the configuration.
- """
- with self.m_lock:
- if self.m_cfg.enable and not new_cfg.enable:
- self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
- elif not self.m_cfg.enable and new_cfg.enable:
- self._log("Config dynamically updated: Group ENABLED.")
- else:
- self._log("Config dynamically updated: Parameters refreshed.")
-
- self.m_cfg = new_cfg
- def get_status(self) -> GroupStatus:
- with self.m_lock:
- tasks = list(self.m_tasks)
- gid = self.m_cfg.identifier
- enable = self.m_cfg.enable
- plugin_name = self.m_cfg.plugin_config.plugin_name
- plugins_status = []
- healthy_count = 0
- for t in tasks:
- if t and t.instance:
- st = t.instance.get_plugin_status()
- if st.is_healthy:
- healthy_count += 1
- plugins_status.append(st)
- return GroupStatus(
- group_id=gid,
- enable=enable,
- plugin_name=plugin_name,
- total_tasks=len(tasks),
- healthy_tasks=healthy_count,
- plugins=plugins_status,
- updated_at=utc_now_str(),
- )
- @start_once
- def start(self):
- if not self.m_cfg.enable:
- return
- self._log("Starting Sentinel...")
- plugin_name = self.m_cfg.plugin_config.plugin_name
- class_name = "".join(part.title() for part in plugin_name.split('_'))
- plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
- self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
- for target in (self._monitor_loop, self._creator_loop):
- thread = threading.Thread(target=target, daemon=True)
- self._background_threads.append(thread)
- thread.start()
- @serialized_stop
- def stop(self):
- self._log("Stopping Sentinel...")
- self.m_stop_event.set()
- deadline = time.monotonic() + self.m_cfg.shutdown_timeout
- self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
- join_threads(self._background_threads, max(0, deadline - time.monotonic()))
- with self.m_lock:
- for task in self.m_tasks:
- task.removed = True
- self._resources.track(task.instance)
- self._resources.close(self.m_cfg.cleanup_timeout)
- threads_done = self._executor.shutdown(timeout=0.1) and join_threads(self._background_threads, 0.1)
- if not self._resources.empty or not threads_done:
- raise RuntimeError('Sentinel shutdown incomplete; resources retained for retry')
- with self.m_lock:
- self.m_tasks.clear()
- self.redis_client.close()
- def _cleanup_instance(self, instance, reason: str = ""):
- if instance is None:
- return True
- cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
- if not cleaned:
- self._log(f"Cleanup retained for retry ({reason})")
- return cleaned
- def _cleanup_task(self, task: Task, reason: str):
- if task and task.instance:
- with task._operation_lock:
- self._cleanup_instance(task.instance, reason)
- def _remove_task(self, task: Task, reason: str):
- removed = False
- with self.m_lock:
- if task in self.m_tasks:
- task.removed = True
- self.m_tasks.remove(task)
- removed = True
- if removed:
- self._cleanup_task(task, reason)
- def _get_redis_key(self, routing_key: str) -> str:
- return f"vs:signal:{routing_key}"
- def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
- """Safely releases account: resets cloud cooldown to 0 to prevent 3-hour lockouts caused by non-account issues."""
- if acc and acc.id > 0 and acc.username not in ("", "Guest"):
- try:
- CloudApi.Instance().lock_account(acc.id, 0)
- self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
- except Exception as e:
- self._log(f"Failed to release account [{acc.username}]: {e}")
- def _disable_account_safely(self, acc: Optional[Account], reason: str = "") -> None:
- """Safely disables account in cloud: sets status='disable', locks for 10 years, and records extra_data."""
- if not acc or acc.id <= 0 or acc.username in ("", "Guest"):
- return
- try:
- cloud = CloudApi.Instance()
- url = f"{cloud.base_url}/api/account/update"
- params = {"account_id": acc.id}
- far_future = (datetime.now(timezone.utc) + timedelta(days=3650)).isoformat(timespec="milliseconds").replace("+00:00", "Z")
- payload = {
- "status": "disable",
- "next_use_time": far_future,
- "extra_data": {
- "disable_reason": reason,
- "disabled_at": int(time.time()),
- "module": "SENTINEL",
- "group": getattr(self.m_cfg, "identifier", "")
- }
- }
- cloud._perform_request('PUT', url, params=params, json_data=payload, headers=cloud._get_headers(), timeout=10)
- self._log(f"🚫 [Account Disabled] Account [{acc.username}] (ID={acc.id}) permanently marked as 'disable' in Cloud API. Reason: {reason}")
- except Exception as e:
- self._log(f"Failed to set account status to disable via cloud API: {e}. Falling back to 10-year lock...")
- try:
- CloudApi.Instance().lock_account(acc.id, 86400 * 3650)
- except Exception as lock_err:
- self._log(f"Fallback 10-year lock also failed: {lock_err}")
- def _dispatch_error_policy(
- self,
- action: str,
- err: Exception,
- task: Optional[Task] = None,
- plg_cfg: Optional[PluginConfig] = None
- ) -> None:
- """
- Sentinel error dispatch and self-healing policy engine:
- Executes self-healing actions based on the (action, error type) matrix:
- - AccountStateError (invalid credentials, banned, incomplete): Permanently disables pool account, never releases back to pool.
- - 429 during spawn/login: Account-level rate limit. Locks account and sets m_spawn_cooldown_until backoff.
- - 429 during query: Session is valid; egress IP frequency limit. Preserves account, removes task to trigger creator spawning a new instance with a fresh proxy.
- - 401 / session expired: Removes broken task.
- - 403 Forbidden / WAF: IP blocked. Removes task to rotate proxy; safely releases account during spawn.
- - Handles logging and timer backoffs internally.
- """
- biz_err = normalize_error(err)
- active_plg_cfg = plg_cfg or (task.instance.config if task else None)
- acc = active_plg_cfg.account if active_plg_cfg else None
- # 0. Fatal Account State Error (credentials invalid, banned, incomplete, not found)
- if isinstance(biz_err, AccountStateError):
- self._log(f"🚨 [Account Fatal Error] {biz_err.__class__.__name__} during {action}: {biz_err.message} (code={biz_err.code})")
- if acc and acc.id > 0 and acc.username not in ("", "Guest"):
- self._disable_account_safely(acc, reason=f"{biz_err.__class__.__name__}: {biz_err.message}")
- # Never release account back to pool
- if task:
- self._remove_task(task, f"account fatal error: {biz_err.message}")
- if action in ("spawn", "login", "create_session"):
- self.m_spawn_cooldown_until = time.time() + 5.0
- # 1. Target site rate limit / 429
- elif isinstance(biz_err, RateLimiteddError):
- if action in ("spawn", "login", "create_session"):
- backoff = float(self.m_cfg.login_backoff)
- self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
- if acc and acc.username not in ("", "Guest") and acc.id > 0:
- try:
- CloudApi.Instance().lock_account(acc.id, int(backoff))
- self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
- except Exception as lock_err:
- self._log(f"Failed to lock account via cloud API: {lock_err}")
- self.m_spawn_cooldown_until = time.time() + backoff
- if task:
- self._remove_task(task, "spawn account rate limited")
- elif action == "query":
- # 429 during query: egress IP throttled, preserve account, remove current task to trigger auto-rotation to a new proxy
- self._log("🔄 [Proxy Rate Limited] Sentinel query hit 429 (IP throttled, Account preserved). Removing task to rotate proxy...")
- if task:
- self._remove_task(task, "query ip rate limited")
- else:
- self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
- if task:
- task.next_run = time.time() + 10.0
- # 2. Session expired / 401
- elif isinstance(biz_err, SessionExpiredOrInvalidError):
- if action in ("spawn", "login", "create_session"):
- self._log(f"❌ [Auth Failed] Sentinel spawn failed (401). Cooling down spawn for 5s...")
- self.m_spawn_cooldown_until = time.time() + 5.0
- else:
- self._log(f"🔄 [Session Expired] Sentinel session expired during {action}. Removing task...")
- if task:
- self._remove_task(task, f"{action} session expired")
- # 3. Permission denied / WAF blocked / 403
- elif isinstance(biz_err, PermissionDeniedError):
- proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
- self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
- if task:
- self._remove_task(task, f"{action} waf blocked")
- if action in ("spawn", "login", "create_session"):
- self._release_account_safely(acc, reason="WAF Blocked")
- self.m_spawn_cooldown_until = time.time() + proxy_cd
- # 4. Network or browser timeout
- elif isinstance(biz_err, NetworkTimeoutError):
- self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
- if task:
- task.next_run = time.time() + 5.0
- if action in ("spawn", "login", "create_session"):
- self._release_account_safely(acc, reason="Network Timeout")
- self.m_spawn_cooldown_until = time.time() + 10.0
- # 5. Fallback policy
- else:
- self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
- if action in ("spawn", "login", "create_session"):
- self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
- self.m_spawn_cooldown_until = time.time() + 10.0
- elif task:
- task.next_run = time.time() + 10.0
- # 6. Proxy health metrics recording
- if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
- ProxyManager.Instance().mark_proxy_result(
- proxy=active_plg_cfg.proxy.model_dump(),
- plugin_name=self.m_cfg.plugin_config.plugin_name,
- action=action,
- success=False,
- http_code=biz_err.http_status,
- error_type=biz_err.message
- )
- def _is_within_active_hours(self) -> bool:
- """
- Checks if current time is within allowed operating hours (UTC).
- """
- start_str = self.m_cfg.active_time_start
- end_str = self.m_cfg.active_time_end
- if (start_str == "00:00" and end_str == "23:59") or not start_str or not end_str:
- return True
- current_utc_time = datetime.now(timezone.utc).time()
- start_time = datetime.strptime(start_str, "%H:%M").time()
- end_time = datetime.strptime(end_str, "%H:%M").time()
- if start_time <= end_time:
- return start_time <= current_utc_time <= end_time
- else:
- return current_utc_time >= start_time or current_utc_time <= end_time
- def _monitor_task(self, task, apt_type):
- try:
- with task._operation_lock, self._resources.operation(task.instance):
- if self.m_stop_event.is_set() or task.removed:
- return
- now = time.time()
- if now >= task.next_health_check:
- if not task.instance.health_check():
- self._remove_task(task, "health check failed")
- return
- task.next_health_check = now + 10
- if apt_type is None:
- return
- qw = self.m_cfg.sentinel.query_wait
- if qw.mode == QueryWaitMode.Loop:
- interval = 1
- elif qw.mode == QueryWaitMode.Fixed:
- interval = qw.fixed_wait
- else:
- interval = random.uniform(qw.random_min, qw.random_max)
- result = task.instance.query(apt_type)
- result.apt_type = apt_type
- with self.m_lock:
- notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
- if result.success:
- payload = {"group_id": self.m_cfg.identifier,
- "apt_type": apt_type.model_dump(),
- "query_result": result.to_snapshot_payload(),
- "timestamp": time.time()}
- self.redis_client.setex(self._get_redis_key(apt_type.routing_key),
- self.m_cfg.sentinel.signal_ttl, json.dumps(payload))
- task.next_run = time.time() + max(0.05, interval)
- if task.instance.config.proxy.port > 0:
- ProxyManager.Instance().mark_proxy_result(
- proxy=task.instance.config.proxy.model_dump(),
- plugin_name=self.m_cfg.plugin_config.plugin_name,
- action="query", success=True, http_code=200)
- except Exception as exc:
- self._dispatch_error_policy("query", exc, task=task)
- finally:
- with self.m_lock:
- task.busy = False
- def _schedule_monitor_jobs(self):
- with self.m_lock:
- tasks = sorted(self.m_tasks, key=lambda t: t.next_run)
- gap = max(0.05, self._get_average_interval() / max(1, len(tasks)))
- for task in tasks:
- if task.busy or task.removed or self.m_stop_event.is_set():
- continue
- now = time.time()
- apt_type = None
- apt_types = self.m_cfg.appointment_types
- if (apt_types and now >= task.next_run
- and now - self.m_last_group_query_time >= gap):
- weights = [max(0, apt.weight) for apt in apt_types]
- apt_type = (random.choices(apt_types, weights=weights, k=1)[0]
- if sum(weights) else random.choice(apt_types))
- if apt_type is None and now < task.next_health_check:
- continue
- task.busy = True
- if self._executor.submit(self._monitor_task, task, apt_type):
- if apt_type is not None:
- self.m_last_group_query_time = now
- else:
- task.busy = False
- def _monitor_loop(self):
- self._log("Monitor loop started.")
- while not self.m_stop_event.wait(0.05):
- try:
- if self.m_cfg.enable and self._is_within_active_hours():
- self._schedule_monitor_jobs()
- except Exception as exc:
- self._dispatch_error_policy("monitor_loop", exc)
- def _creator_loop(self):
- self._log("Creator loop started.")
- while not self.m_stop_event.wait(1.0):
- if not self._is_within_active_hours():
- continue
- if time.time() < self.m_spawn_cooldown_until:
- continue
-
- with self.m_lock:
- current = len(self.m_tasks)
- target = self.m_cfg.sentinel.target_instances
-
- if current < target:
- self._spawn_sentinel_worker()
- def _spawn_sentinel_worker(self):
- if self.m_stop_event.is_set() or not self._resources.retry_failed():
- return
- instance = None
- task = None
- success = False
- plg_cfg = None
- try:
- plg_cfg = PluginConfig()
- plg_cfg.debug = self.m_cfg.debug
- plg_cfg.free_config = self.m_cfg.free_config
- plg_cfg.session_max_life = self.m_cfg.session_max_life
-
- if not self.m_cfg.need_account:
- plg_cfg.account.id = 0
- plg_cfg.account.username = "Guest"
- else:
- acc = CloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
- plg_cfg.account = type(plg_cfg.account)(**acc)
-
- if self.m_cfg.need_proxy:
- proxy = ProxyManager.Instance().get_next_proxy(
- self.m_cfg.proxy_pool,
- self.m_cfg.proxy_cd,
- plugin_name=self.m_cfg.plugin_config.plugin_name
- )
- plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
- instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
- self._resources.track(instance)
- with self._resources.operation(instance):
- instance.set_log(self.m_logger)
- instance.set_config(plg_cfg)
- instance.create_session()
-
- task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
- with self.m_lock:
- if self.m_stop_event.is_set():
- return
- self.m_tasks.append(task)
- success = True
- if plg_cfg.proxy.port > 0:
- ProxyManager.Instance().mark_proxy_result(
- proxy=plg_cfg.proxy.model_dump(),
- plugin_name=self.m_cfg.plugin_config.plugin_name,
- action="spawn",
- success=True,
- http_code=200
- )
- self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
- except Exception as e:
- self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
- finally:
- if not success and instance:
- self._cleanup_instance(instance, "spawn failed")
|