proxy_manager.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. import os
  2. import time
  3. import json
  4. import yaml
  5. import random
  6. import threading
  7. import collections
  8. import concurrent.futures
  9. from typing import List, Dict, Any, Optional
  10. import requests
  11. import configure
  12. from toolkit.cloud_api import CloudApi
  13. from toolkit.task_executor import submit_background
  14. from core_types import BizLogicError
  15. from logger_setup import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
  16. class ProxyManager:
  17. """
  18. Unified proxy scheduling and reputation management hub.
  19. Supports:
  20. - Explicit prefix routing: "local:<pool_name>" / "cloud:<pool_name>"
  21. - Egress IP resolver and cache
  22. - Local channel and cloud reputation intersection weighted scheduling (85% preferred + 15% exploration)
  23. - Five-layer avalanche guardrails (site-wide outage freezing, 20% minimum pool guarantee, half-open revival, hard fallback)
  24. - Asynchronous metric tracking and upstream reporting (fail-silent)
  25. """
  26. _instance = None
  27. def __new__(cls, *args, **kwargs):
  28. if cls._instance is None:
  29. cls._instance = super(ProxyManager, cls).__new__(cls)
  30. cls._instance._init_manager()
  31. return cls._instance
  32. @classmethod
  33. def Instance(cls) -> "ProxyManager":
  34. return cls()
  35. def _init_manager(self):
  36. self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  37. self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
  38. self._lock = threading.RLock()
  39. # Port -> public egress IP cache and node name cache
  40. self._port_to_egress: Dict[int, str] = {}
  41. self._port_to_name: Dict[int, str] = {}
  42. self._probing_ports = set()
  43. # Per-plugin local temporary quarantine blacklist: { plugin_name: { egress_ip: expire_ts } }
  44. self._quarantine: Dict[str, Dict[str, float]] = {}
  45. # Per-plugin cloud reputation cache: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
  46. self._reputation_cache: Dict[str, Dict[str, Any]] = {}
  47. self._fetching_reputation = set()
  48. # Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
  49. self._sliding_history: Dict[str, collections.deque] = {}
  50. def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
  51. if not os.path.exists(self.local_proxies_file):
  52. raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
  53. try:
  54. with open(self.local_proxies_file, "r", encoding="utf-8") as f:
  55. return yaml.safe_load(f) or {}
  56. except Exception as e:
  57. raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
  58. # ==========================================
  59. # Egress IP probe and cache (Egress Resolver)
  60. # ==========================================
  61. def get_cached_egress_ip(self, port: int) -> Optional[str]:
  62. with self._lock:
  63. return self._port_to_egress.get(port)
  64. def invalidate_egress_cache(self, port: int):
  65. """Clears cache when node egress is suspected to have drifted."""
  66. with self._lock:
  67. if port in self._port_to_egress:
  68. old_ip = self._port_to_egress.pop(port, None)
  69. VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
  70. def _async_probe_egress_ip(self, port: int, proto: str = "http"):
  71. """Asynchronously probes the true public egress IP of the port in background, non-blocking."""
  72. with self._lock:
  73. if port in self._port_to_egress or port in self._probing_ports:
  74. return
  75. self._probing_ports.add(port)
  76. def _do_probe():
  77. try:
  78. host = os.getenv("PROXY_HOST") or "127.0.0.1"
  79. proxy_url = f"{proto}://{host}:{port}"
  80. proxies = {"http": proxy_url, "https": proxy_url}
  81. # Try lightweight public IP endpoint
  82. resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
  83. if resp.status_code == 200 and resp.text.strip():
  84. ip = resp.text.strip()
  85. with self._lock:
  86. self._port_to_egress[port] = ip
  87. VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
  88. except Exception:
  89. pass
  90. finally:
  91. with self._lock:
  92. self._probing_ports.discard(port)
  93. if not submit_background(_do_probe):
  94. t = threading.Thread(target=_do_probe, daemon=True)
  95. t.start()
  96. def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
  97. """Gets the IP identifier of the node (prefers true egress IP; falls back to port anchor if unparsed)."""
  98. port = int(node.get("port", 0))
  99. configured_egress = node.get("egress_ip")
  100. if configured_egress:
  101. return configured_egress
  102. with self._lock:
  103. cached = self._port_to_egress.get(port)
  104. if cached:
  105. return cached
  106. if trigger_probe and port:
  107. proto = node.get("proto", "http")
  108. self._async_probe_egress_ip(port, proto)
  109. return f"port_{port}"
  110. # ==========================================
  111. # Cloud reputation sync and avalanche guardrails
  112. # ==========================================
  113. def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
  114. """Gets cloud reputation mapping (asynchronously refreshed in background without blocking)."""
  115. now = time.time()
  116. with self._lock:
  117. cache_entry = self._reputation_cache.get(plugin_name)
  118. if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
  119. return cache_entry.get("map", {})
  120. if plugin_name in self._fetching_reputation:
  121. return (cache_entry.get("map", {}) if cache_entry else {})
  122. self._fetching_reputation.add(plugin_name)
  123. def _fetch_worker():
  124. try:
  125. rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
  126. with self._lock:
  127. self._reputation_cache[plugin_name] = {
  128. "last_fetch": time.time(),
  129. "map": rep_map or {}
  130. }
  131. except Exception:
  132. pass
  133. finally:
  134. with self._lock:
  135. self._fetching_reputation.discard(plugin_name)
  136. if not submit_background(_fetch_worker):
  137. threading.Thread(target=_fetch_worker, daemon=True).start()
  138. return (cache_entry.get("map", {}) if cache_entry else {})
  139. def _is_site_outage(self, plugin_name: str) -> bool:
  140. """
  141. Layer 1 protection: Site-wide outage determination
  142. Within the past 180s, across >= 2 distinct IPs with sample size >= 5:
  143. if failure rate >= 90% or >= 70% 502/503/520~526 errors, judge as site-wide outage.
  144. """
  145. with self._lock:
  146. history = self._sliding_history.get(plugin_name)
  147. if not history or len(history) < 5:
  148. return False
  149. now = time.time()
  150. recent_events = [e for e in history if now - e["ts"] < 180]
  151. if len(recent_events) < 5:
  152. return False
  153. distinct_ips = {e["egress_ip"] for e in recent_events}
  154. if len(distinct_ips) < 2:
  155. return False
  156. failures = [e for e in recent_events if not e["success"]]
  157. fail_rate = len(failures) / len(recent_events)
  158. # Count of severe server-side errors
  159. server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
  160. if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
  161. return True
  162. return False
  163. # ==========================================
  164. # Core scheduling logic (intersection weighting + 5 guardrails)
  165. # ==========================================
  166. def get_next_proxy(
  167. self,
  168. pools: List[str],
  169. proxy_cd: int = 60,
  170. plugin_name: Optional[str] = None
  171. ) -> Dict[str, Any]:
  172. """
  173. Resolves and acquires the next proxy based on pools list.
  174. @param pools: Proxy pool list, e.g. ["local:local_multi_pool"] or ["cloud:proxy-cheap"]
  175. @param proxy_cd: Proxy cooldown time (used for cloud)
  176. @param plugin_name: Plugin name (e.g. 'vfs_plugin'), used for reputation scoring and quarantine
  177. @return dict Standard proxy info (contains id, proto, ip, port, username, password, egress_ip, name)
  178. """
  179. if getattr(configure, "TEST_PROXY", None):
  180. return configure.TEST_PROXY
  181. if not pools:
  182. raise BizLogicError(message="proxy_pool configuration list is empty.")
  183. local_targets = []
  184. cloud_targets = []
  185. for pool_item in pools:
  186. if not isinstance(pool_item, str):
  187. continue
  188. pool_item = pool_item.strip()
  189. if pool_item.startswith("local:"):
  190. local_pool_name = pool_item[len("local:"):].strip()
  191. if local_pool_name:
  192. local_targets.append(local_pool_name)
  193. elif pool_item.startswith("cloud:"):
  194. cloud_pool_name = pool_item[len("cloud:"):].strip()
  195. if cloud_pool_name:
  196. cloud_targets.append(cloud_pool_name)
  197. else:
  198. raise BizLogicError(
  199. message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
  200. )
  201. if local_targets:
  202. all_local_data = self._load_local_proxies()
  203. nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
  204. nodes_map = {
  205. n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
  206. }
  207. pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
  208. candidate_proxies: List[Dict[str, Any]] = []
  209. for lt in local_targets:
  210. pool_items = pools_dict.get(lt, [])
  211. if pool_items and isinstance(pool_items, list):
  212. for item in pool_items:
  213. if isinstance(item, str):
  214. node_obj = nodes_map.get(item)
  215. if node_obj:
  216. candidate_proxies.append(node_obj)
  217. elif isinstance(item, dict):
  218. candidate_proxies.append(item)
  219. if not candidate_proxies:
  220. raise BizLogicError(
  221. message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
  222. )
  223. # Record node name cache
  224. for node in candidate_proxies:
  225. port = int(node.get("port", 0))
  226. if port and "name" in node:
  227. self._port_to_name[port] = node["name"]
  228. # If no plugin_name is specified, fallback to classic random round-robin
  229. if not plugin_name:
  230. chosen = random.choice(candidate_proxies)
  231. return self._format_proxy_dict(chosen)
  232. # ------------------------------------------------------------------
  233. # Intelligent tiered scheduling (reputation + tiered routing + 5 guardrails)
  234. # ------------------------------------------------------------------
  235. now = time.time()
  236. rep_map = self._get_reputation_map(plugin_name)
  237. with self._lock:
  238. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  239. # Clean up expired quarantine entries
  240. expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
  241. for ip in expired_ips:
  242. quarantine_map.pop(ip, None)
  243. # Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
  244. total_nodes_count = len(candidate_proxies)
  245. quarantined_count = sum(
  246. 1 for node in candidate_proxies
  247. if self._resolve_ip_identifier(node) in quarantine_map
  248. )
  249. # If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
  250. if total_nodes_count > 0:
  251. available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
  252. if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
  253. VSC_WARN("vs_proxy", f"[Proxy Anti-Avalanche] Plugin {plugin_name} available nodes below floor threshold (availability {available_ratio:.1%}). Clearing quarantine list!")
  254. quarantine_map.clear()
  255. # Divide into tiers
  256. trusted_tier: List[Dict[str, Any]] = []
  257. explore_tier: List[Dict[str, Any]] = []
  258. for node in candidate_proxies:
  259. ip_id = self._resolve_ip_identifier(node)
  260. # Check if in quarantine
  261. if ip_id in quarantine_map:
  262. continue
  263. # Check cloud reputation
  264. rep_info = rep_map.get(ip_id)
  265. if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
  266. trusted_tier.append(node)
  267. else:
  268. explore_tier.append(node)
  269. # Tier weighted scheduling: 85% preferred, 15% exploration
  270. chosen = None
  271. if trusted_tier and explore_tier:
  272. if random.random() < 0.85:
  273. chosen = random.choice(trusted_tier)
  274. else:
  275. chosen = random.choice(explore_tier)
  276. elif trusted_tier:
  277. chosen = random.choice(trusted_tier)
  278. elif explore_tier:
  279. chosen = random.choice(explore_tier)
  280. # Layer 5 protection: Hard fallback (if empty after filtering, fallback to full pool random)
  281. if not chosen:
  282. VSC_WARN("vs_proxy", f"[Proxy Hard Fallback] Plugin {plugin_name} has no available nodes after reputation filter. Triggering full random fallback!")
  283. chosen = random.choice(candidate_proxies)
  284. return self._format_proxy_dict(chosen)
  285. if cloud_targets:
  286. return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
  287. raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
  288. def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
  289. """Formats and normalizes the proxy dictionary."""
  290. port = int(chosen.get("port", 80))
  291. egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
  292. proxy_ip = os.getenv("PROXY_HOST") or chosen.get("ip", "127.0.0.1")
  293. return {
  294. "id": chosen.get("id", 0),
  295. "proto": chosen.get("proto", "http"),
  296. "ip": proxy_ip,
  297. "port": port,
  298. "username": chosen.get("username", ""),
  299. "password": chosen.get("password", ""),
  300. "name": chosen.get("name", self._port_to_name.get(port, "")),
  301. "egress_ip": egress_ip,
  302. }
  303. # ==========================================
  304. # Telemetry upstream and real-time feedback
  305. # ==========================================
  306. def mark_proxy_result(
  307. self,
  308. proxy: Dict[str, Any],
  309. plugin_name: str,
  310. action: str,
  311. success: bool,
  312. http_code: int = 200,
  313. error_type: str = "",
  314. latency_ms: int = 0
  315. ):
  316. """
  317. Marks proxy usage outcome and reports asynchronously.
  318. Includes:
  319. - Site-wide outage determination
  320. - Targeted block detection and temporary quarantine
  321. - Half-open revival
  322. - Drift detection
  323. """
  324. if not proxy or not plugin_name:
  325. return
  326. port = int(proxy.get("port", 0))
  327. egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
  328. node_name = proxy.get("name") or self._port_to_name.get(port, "")
  329. # 1. Record into sliding call history
  330. now = time.time()
  331. with self._lock:
  332. history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
  333. history.append({
  334. "ts": now,
  335. "egress_ip": egress_ip,
  336. "success": success,
  337. "http_code": http_code,
  338. "error_type": error_type
  339. })
  340. # 2. Check if currently in site-wide outage
  341. is_outage = self._is_site_outage(plugin_name)
  342. with self._lock:
  343. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  344. if success:
  345. # Success event: If quarantined or half-open, immediately revive and unquarantine
  346. if egress_ip in quarantine_map:
  347. quarantine_map.pop(egress_ip, None)
  348. VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
  349. else:
  350. # Failure event
  351. if is_outage:
  352. VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
  353. else:
  354. # Layer 2: Detailed error classification
  355. is_waf_block = (
  356. "cf-mitigated" in error_type.lower() or
  357. "403201" in error_type or
  358. "403202" in error_type or
  359. "waf blocked" in error_type.lower() or
  360. (http_code == 403 and "mitigated" in error_type.lower())
  361. )
  362. if is_waf_block:
  363. # Targeted block: enter quarantine list (default 1800s)
  364. quarantine_map[egress_ip] = now + 1800
  365. VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF block. Quarantined for 30 minutes!")
  366. # Trigger potential drift detection: invalidate cache to re-probe true public IP
  367. self.invalidate_egress_cache(port)
  368. # 3. Asynchronously report telemetry metrics to cloud (Fail-Silent)
  369. metric_payload = {
  370. "client_id": getattr(configure, "NODE_ID", "local_node"),
  371. "plugin_name": plugin_name,
  372. "local_port": port,
  373. "node_name": node_name,
  374. "egress_ip": egress_ip,
  375. "action": action,
  376. "success": success,
  377. "http_code": http_code,
  378. "error_type": error_type,
  379. "latency_ms": latency_ms,
  380. "timestamp": int(now)
  381. }
  382. def _report_worker():
  383. try:
  384. CloudApi.Instance().report_proxy_metric(metric_payload)
  385. except Exception:
  386. pass
  387. submit_background(_report_worker)