| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453 |
- import os
- import time
- import json
- import yaml
- import random
- import threading
- import collections
- import concurrent.futures
- from typing import List, Dict, Any, Optional
- import requests
- import configure
- from toolkit.cloud_api import CloudApi
- from core_types import BizLogicError
- from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
- class ProxyManager:
- """
- Unified proxy scheduling and reputation management hub.
- Supports:
- - Explicit prefix routing: "local:<pool_name>" / "cloud:<pool_name>"
- - Egress IP resolver and cache
- - Local channel and cloud reputation intersection weighted scheduling (85% preferred + 15% exploration)
- - Five-layer avalanche guardrails (site-wide outage freezing, 20% minimum pool guarantee, half-open revival, hard fallback)
- - Asynchronous metric tracking and upstream reporting (fail-silent)
- """
- _instance = None
- def __new__(cls, *args, **kwargs):
- if cls._instance is None:
- cls._instance = super(ProxyManager, cls).__new__(cls)
- cls._instance._init_manager()
- return cls._instance
- @classmethod
- def Instance(cls) -> "ProxyManager":
- return cls()
- def _init_manager(self):
- self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
- self._lock = threading.RLock()
- # Port -> public egress IP cache and node name cache
- self._port_to_egress: Dict[int, str] = {}
- self._port_to_name: Dict[int, str] = {}
- self._probing_ports = set()
- # Per-plugin local temporary quarantine blacklist: { plugin_name: { egress_ip: expire_ts } }
- self._quarantine: Dict[str, Dict[str, float]] = {}
- # Per-plugin cloud reputation cache: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
- self._reputation_cache: Dict[str, Dict[str, Any]] = {}
- self._fetching_reputation = set()
- # Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
- self._sliding_history: Dict[str, collections.deque] = {}
- def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
- if not os.path.exists(self.local_proxies_file):
- raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
- try:
- with open(self.local_proxies_file, "r", encoding="utf-8") as f:
- return yaml.safe_load(f) or {}
- except Exception as e:
- raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
- # ==========================================
- # Egress IP probe and cache (Egress Resolver)
- # ==========================================
- def get_cached_egress_ip(self, port: int) -> Optional[str]:
- with self._lock:
- return self._port_to_egress.get(port)
- def invalidate_egress_cache(self, port: int):
- """Clears cache when node egress is suspected to have drifted."""
- with self._lock:
- if port in self._port_to_egress:
- old_ip = self._port_to_egress.pop(port, None)
- VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
- def _async_probe_egress_ip(self, port: int, proto: str = "http"):
- """Asynchronously probes the true public egress IP of the port in background, non-blocking."""
- with self._lock:
- if port in self._port_to_egress or port in self._probing_ports:
- return
- self._probing_ports.add(port)
- def _do_probe():
- try:
- host = os.getenv("PROXY_HOST") or "127.0.0.1"
- proxy_url = f"{proto}://{host}:{port}"
- proxies = {"http": proxy_url, "https": proxy_url}
- # Try lightweight public IP endpoint
- resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
- if resp.status_code == 200 and resp.text.strip():
- ip = resp.text.strip()
- with self._lock:
- self._port_to_egress[port] = ip
- VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
- except Exception:
- pass
- finally:
- with self._lock:
- self._probing_ports.discard(port)
- t = threading.Thread(target=_do_probe, daemon=True)
- t.start()
- def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
- """Gets the IP identifier of the node (prefers true egress IP; falls back to port anchor if unparsed)."""
- port = int(node.get("port", 0))
- configured_egress = node.get("egress_ip")
- if configured_egress:
- return configured_egress
- with self._lock:
- cached = self._port_to_egress.get(port)
- if cached:
- return cached
- if trigger_probe and port:
- proto = node.get("proto", "http")
- self._async_probe_egress_ip(port, proto)
- return f"port_{port}"
- # ==========================================
- # Cloud reputation sync and avalanche guardrails
- # ==========================================
- def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
- """Gets cloud reputation mapping (asynchronously refreshed in background without blocking)."""
- now = time.time()
- with self._lock:
- cache_entry = self._reputation_cache.get(plugin_name)
- if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
- return cache_entry.get("map", {})
- if plugin_name in self._fetching_reputation:
- return (cache_entry.get("map", {}) if cache_entry else {})
- self._fetching_reputation.add(plugin_name)
- def _fetch_worker():
- try:
- rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
- with self._lock:
- self._reputation_cache[plugin_name] = {
- "last_fetch": time.time(),
- "map": rep_map or {}
- }
- except Exception:
- pass
- finally:
- with self._lock:
- self._fetching_reputation.discard(plugin_name)
- threading.Thread(target=_fetch_worker, daemon=True).start()
- return (cache_entry.get("map", {}) if cache_entry else {})
- def _is_site_outage(self, plugin_name: str) -> bool:
- """
- Layer 1 protection: Site-wide outage determination
- Within the past 180s, across >= 2 distinct IPs with sample size >= 5:
- if failure rate >= 90% or >= 70% 502/503/520~526 errors, judge as site-wide outage.
- """
- with self._lock:
- history = self._sliding_history.get(plugin_name)
- if not history or len(history) < 5:
- return False
- now = time.time()
- recent_events = [e for e in history if now - e["ts"] < 180]
- if len(recent_events) < 5:
- return False
- distinct_ips = {e["egress_ip"] for e in recent_events}
- if len(distinct_ips) < 2:
- return False
- failures = [e for e in recent_events if not e["success"]]
- fail_rate = len(failures) / len(recent_events)
- # Count of severe server-side errors
- server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
- if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
- return True
- return False
- # ==========================================
- # Core scheduling logic (intersection weighting + 5 guardrails)
- # ==========================================
- def get_next_proxy(
- self,
- pools: List[str],
- proxy_cd: int = 60,
- plugin_name: Optional[str] = None
- ) -> Dict[str, Any]:
- """
- Resolves and acquires the next proxy based on pools list.
- @param pools: Proxy pool list, e.g. ["local:local_multi_pool"] or ["cloud:proxy-cheap"]
- @param proxy_cd: Proxy cooldown time (used for cloud)
- @param plugin_name: Plugin name (e.g. 'vfs_plugin'), used for reputation scoring and quarantine
- @return dict Standard proxy info (contains id, proto, ip, port, username, password, egress_ip, name)
- """
- if getattr(configure, "TEST_PROXY", None):
- return configure.TEST_PROXY
- if not pools:
- raise BizLogicError(message="proxy_pool configuration list is empty.")
- local_targets = []
- cloud_targets = []
- for pool_item in pools:
- if not isinstance(pool_item, str):
- continue
- pool_item = pool_item.strip()
- if pool_item.startswith("local:"):
- local_pool_name = pool_item[len("local:"):].strip()
- if local_pool_name:
- local_targets.append(local_pool_name)
- elif pool_item.startswith("cloud:"):
- cloud_pool_name = pool_item[len("cloud:"):].strip()
- if cloud_pool_name:
- cloud_targets.append(cloud_pool_name)
- else:
- raise BizLogicError(
- message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
- )
- if local_targets:
- all_local_data = self._load_local_proxies()
- nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
- nodes_map = {
- n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
- }
- pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
- candidate_proxies: List[Dict[str, Any]] = []
- for lt in local_targets:
- pool_items = pools_dict.get(lt, [])
- if pool_items and isinstance(pool_items, list):
- for item in pool_items:
- if isinstance(item, str):
- node_obj = nodes_map.get(item)
- if node_obj:
- candidate_proxies.append(node_obj)
- elif isinstance(item, dict):
- candidate_proxies.append(item)
- if not candidate_proxies:
- raise BizLogicError(
- message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
- )
- # Record node name cache
- for node in candidate_proxies:
- port = int(node.get("port", 0))
- if port and "name" in node:
- self._port_to_name[port] = node["name"]
- # If no plugin_name is specified, fallback to classic random round-robin
- if not plugin_name:
- chosen = random.choice(candidate_proxies)
- return self._format_proxy_dict(chosen)
- # ------------------------------------------------------------------
- # Intelligent tiered scheduling (reputation + tiered routing + 5 guardrails)
- # ------------------------------------------------------------------
- now = time.time()
- rep_map = self._get_reputation_map(plugin_name)
- with self._lock:
- quarantine_map = self._quarantine.setdefault(plugin_name, {})
- # Clean up expired quarantine entries
- expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
- for ip in expired_ips:
- quarantine_map.pop(ip, None)
- # Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
- total_nodes_count = len(candidate_proxies)
- quarantined_count = sum(
- 1 for node in candidate_proxies
- if self._resolve_ip_identifier(node) in quarantine_map
- )
- # If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
- if total_nodes_count > 0:
- available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
- if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
- VSC_WARN("vs_proxy", f"[Proxy Anti-Avalanche] Plugin {plugin_name} available nodes below floor threshold (availability {available_ratio:.1%}). Clearing quarantine list!")
- quarantine_map.clear()
- # Divide into tiers
- trusted_tier: List[Dict[str, Any]] = []
- explore_tier: List[Dict[str, Any]] = []
- for node in candidate_proxies:
- ip_id = self._resolve_ip_identifier(node)
- # Check if in quarantine
- if ip_id in quarantine_map:
- continue
- # Check cloud reputation
- rep_info = rep_map.get(ip_id)
- if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
- trusted_tier.append(node)
- else:
- explore_tier.append(node)
- # Tier weighted scheduling: 85% preferred, 15% exploration
- chosen = None
- if trusted_tier and explore_tier:
- if random.random() < 0.85:
- chosen = random.choice(trusted_tier)
- else:
- chosen = random.choice(explore_tier)
- elif trusted_tier:
- chosen = random.choice(trusted_tier)
- elif explore_tier:
- chosen = random.choice(explore_tier)
- # Layer 5 protection: Hard fallback (if empty after filtering, fallback to full pool random)
- if not chosen:
- VSC_WARN("vs_proxy", f"[Proxy Hard Fallback] Plugin {plugin_name} has no available nodes after reputation filter. Triggering full random fallback!")
- chosen = random.choice(candidate_proxies)
- return self._format_proxy_dict(chosen)
- if cloud_targets:
- return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
- raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
- def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
- """Formats and normalizes the proxy dictionary."""
- port = int(chosen.get("port", 80))
- egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
- proxy_ip = os.getenv("PROXY_HOST") or chosen.get("ip", "127.0.0.1")
- return {
- "id": chosen.get("id", 0),
- "proto": chosen.get("proto", "http"),
- "ip": proxy_ip,
- "port": port,
- "username": chosen.get("username", ""),
- "password": chosen.get("password", ""),
- "name": chosen.get("name", self._port_to_name.get(port, "")),
- "egress_ip": egress_ip,
- }
- # ==========================================
- # Telemetry upstream and real-time feedback
- # ==========================================
- def mark_proxy_result(
- self,
- proxy: Dict[str, Any],
- plugin_name: str,
- action: str,
- success: bool,
- http_code: int = 200,
- error_type: str = "",
- latency_ms: int = 0
- ):
- """
- Marks proxy usage outcome and reports asynchronously.
- Includes:
- - Site-wide outage determination
- - Targeted block detection and temporary quarantine
- - Half-open revival
- - Drift detection
- """
- if not proxy or not plugin_name:
- return
- port = int(proxy.get("port", 0))
- egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
- node_name = proxy.get("name") or self._port_to_name.get(port, "")
- # 1. Record into sliding call history
- now = time.time()
- with self._lock:
- history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
- history.append({
- "ts": now,
- "egress_ip": egress_ip,
- "success": success,
- "http_code": http_code,
- "error_type": error_type
- })
- # 2. Check if currently in site-wide outage
- is_outage = self._is_site_outage(plugin_name)
- with self._lock:
- quarantine_map = self._quarantine.setdefault(plugin_name, {})
- if success:
- # Success event: If quarantined or half-open, immediately revive and unquarantine
- if egress_ip in quarantine_map:
- quarantine_map.pop(egress_ip, None)
- VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
- else:
- # Failure event
- if is_outage:
- VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
- else:
- # Layer 2: Detailed error classification
- is_waf_block = (
- "cf-mitigated" in error_type.lower() or
- "403202" in error_type or
- "waf blocked" in error_type.lower() or
- (http_code == 403 and "mitigated" in error_type.lower())
- )
- if is_waf_block:
- # Targeted block: enter quarantine list (default 1800s)
- quarantine_map[egress_ip] = now + 1800
- VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF block. Quarantined for 30 minutes!")
- # Trigger potential drift detection: invalidate cache to re-probe true public IP
- self.invalidate_egress_cache(port)
- # 3. Asynchronously report telemetry metrics to cloud (Fail-Silent)
- metric_payload = {
- "client_id": getattr(configure, "NODE_ID", "local_node"),
- "plugin_name": plugin_name,
- "local_port": port,
- "node_name": node_name,
- "egress_ip": egress_ip,
- "action": action,
- "success": success,
- "http_code": http_code,
- "error_type": error_type,
- "latency_ms": latency_ms,
- "timestamp": int(now)
- }
-
- def _report_worker():
- try:
- CloudApi.Instance().report_proxy_metric(metric_payload)
- except Exception:
- pass
- t = threading.Thread(target=_report_worker, daemon=True)
- t.start()
|