proxy_manager.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463
  1. import os
  2. import time
  3. import json
  4. import yaml
  5. import random
  6. import threading
  7. import collections
  8. import concurrent.futures
  9. from typing import List, Dict, Any, Optional
  10. import requests
  11. import configure
  12. from toolkit.cloud_api import CloudApi
  13. from toolkit.thread_pool import ThreadPool
  14. from core_types import BizLogicError
  15. from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
  16. class ProxyManager:
  17. """
  18. Unified proxy scheduling and reputation management hub.
  19. Supports:
  20. - Explicit prefix routing: "local:<pool_name>" / "cloud:<pool_name>"
  21. - Egress IP resolver and cache
  22. - Local channel and cloud reputation intersection weighted scheduling (85% preferred + 15% exploration)
  23. - Five-layer avalanche guardrails (site-wide outage freezing, 20% minimum pool guarantee, half-open revival, hard fallback)
  24. - Asynchronous metric tracking and upstream reporting (fail-silent)
  25. """
  26. _instance = None
  27. def __new__(cls, *args, **kwargs):
  28. if cls._instance is None:
  29. cls._instance = super(ProxyManager, cls).__new__(cls)
  30. cls._instance._init_manager()
  31. return cls._instance
  32. @classmethod
  33. def Instance(cls) -> "ProxyManager":
  34. return cls()
  35. def _init_manager(self):
  36. self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  37. self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
  38. self._lock = threading.RLock()
  39. # Port -> public egress IP cache and node name cache
  40. self._port_to_egress: Dict[int, str] = {}
  41. self._port_to_name: Dict[int, str] = {}
  42. self._probing_ports = set()
  43. # Per-plugin local temporary quarantine blacklist: { plugin_name: { egress_ip: expire_ts } }
  44. self._quarantine: Dict[str, Dict[str, float]] = {}
  45. # Per-plugin cloud reputation cache: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
  46. self._reputation_cache: Dict[str, Dict[str, Any]] = {}
  47. self._fetching_reputation = set()
  48. # Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
  49. self._sliding_history: Dict[str, collections.deque] = {}
  50. def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
  51. if not os.path.exists(self.local_proxies_file):
  52. raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
  53. try:
  54. with open(self.local_proxies_file, "r", encoding="utf-8") as f:
  55. return yaml.safe_load(f) or {}
  56. except Exception as e:
  57. raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
  58. # ==========================================
  59. # Egress IP probe and cache (Egress Resolver)
  60. # ==========================================
  61. def get_cached_egress_ip(self, port: int) -> Optional[str]:
  62. with self._lock:
  63. return self._port_to_egress.get(port)
  64. def invalidate_egress_cache(self, port: int):
  65. """Clears cache when node egress is suspected to have drifted."""
  66. with self._lock:
  67. if port in self._port_to_egress:
  68. old_ip = self._port_to_egress.pop(port, None)
  69. VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
  70. def _async_probe_egress_ip(self, port: int, proto: str = "http"):
  71. """Asynchronously probes the true public egress IP of the port in background, non-blocking."""
  72. with self._lock:
  73. if port in self._port_to_egress or port in self._probing_ports:
  74. return
  75. self._probing_ports.add(port)
  76. def _do_probe():
  77. try:
  78. host = os.getenv("PROXY_HOST") or "127.0.0.1"
  79. proxy_url = f"{proto}://{host}:{port}"
  80. proxies = {"http": proxy_url, "https": proxy_url}
  81. # Try lightweight public IP endpoint
  82. resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
  83. if resp.status_code == 200 and resp.text.strip():
  84. ip = resp.text.strip()
  85. with self._lock:
  86. self._port_to_egress[port] = ip
  87. VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
  88. except Exception:
  89. pass
  90. finally:
  91. with self._lock:
  92. self._probing_ports.discard(port)
  93. try:
  94. ThreadPool.getInstance().enqueue(_do_probe)
  95. except Exception:
  96. t = threading.Thread(target=_do_probe, daemon=True)
  97. t.start()
  98. def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
  99. """Gets the IP identifier of the node (prefers true egress IP; falls back to port anchor if unparsed)."""
  100. port = int(node.get("port", 0))
  101. configured_egress = node.get("egress_ip")
  102. if configured_egress:
  103. return configured_egress
  104. with self._lock:
  105. cached = self._port_to_egress.get(port)
  106. if cached:
  107. return cached
  108. if trigger_probe and port:
  109. proto = node.get("proto", "http")
  110. self._async_probe_egress_ip(port, proto)
  111. return f"port_{port}"
  112. # ==========================================
  113. # Cloud reputation sync and avalanche guardrails
  114. # ==========================================
  115. def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
  116. """Gets cloud reputation mapping (asynchronously refreshed in background without blocking)."""
  117. now = time.time()
  118. with self._lock:
  119. cache_entry = self._reputation_cache.get(plugin_name)
  120. if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
  121. return cache_entry.get("map", {})
  122. if plugin_name in self._fetching_reputation:
  123. return (cache_entry.get("map", {}) if cache_entry else {})
  124. self._fetching_reputation.add(plugin_name)
  125. def _fetch_worker():
  126. try:
  127. rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
  128. with self._lock:
  129. self._reputation_cache[plugin_name] = {
  130. "last_fetch": time.time(),
  131. "map": rep_map or {}
  132. }
  133. except Exception:
  134. pass
  135. finally:
  136. with self._lock:
  137. self._fetching_reputation.discard(plugin_name)
  138. try:
  139. ThreadPool.getInstance().enqueue(_fetch_worker)
  140. except Exception:
  141. threading.Thread(target=_fetch_worker, daemon=True).start()
  142. return (cache_entry.get("map", {}) if cache_entry else {})
  143. def _is_site_outage(self, plugin_name: str) -> bool:
  144. """
  145. Layer 1 protection: Site-wide outage determination
  146. Within the past 180s, across >= 2 distinct IPs with sample size >= 5:
  147. if failure rate >= 90% or >= 70% 502/503/520~526 errors, judge as site-wide outage.
  148. """
  149. with self._lock:
  150. history = self._sliding_history.get(plugin_name)
  151. if not history or len(history) < 5:
  152. return False
  153. now = time.time()
  154. recent_events = [e for e in history if now - e["ts"] < 180]
  155. if len(recent_events) < 5:
  156. return False
  157. distinct_ips = {e["egress_ip"] for e in recent_events}
  158. if len(distinct_ips) < 2:
  159. return False
  160. failures = [e for e in recent_events if not e["success"]]
  161. fail_rate = len(failures) / len(recent_events)
  162. # Count of severe server-side errors
  163. server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
  164. if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
  165. return True
  166. return False
  167. # ==========================================
  168. # Core scheduling logic (intersection weighting + 5 guardrails)
  169. # ==========================================
  170. def get_next_proxy(
  171. self,
  172. pools: List[str],
  173. proxy_cd: int = 60,
  174. plugin_name: Optional[str] = None
  175. ) -> Dict[str, Any]:
  176. """
  177. Resolves and acquires the next proxy based on pools list.
  178. @param pools: Proxy pool list, e.g. ["local:local_multi_pool"] or ["cloud:proxy-cheap"]
  179. @param proxy_cd: Proxy cooldown time (used for cloud)
  180. @param plugin_name: Plugin name (e.g. 'vfs_plugin'), used for reputation scoring and quarantine
  181. @return dict Standard proxy info (contains id, proto, ip, port, username, password, egress_ip, name)
  182. """
  183. if getattr(configure, "TEST_PROXY", None):
  184. return configure.TEST_PROXY
  185. if not pools:
  186. raise BizLogicError(message="proxy_pool configuration list is empty.")
  187. local_targets = []
  188. cloud_targets = []
  189. for pool_item in pools:
  190. if not isinstance(pool_item, str):
  191. continue
  192. pool_item = pool_item.strip()
  193. if pool_item.startswith("local:"):
  194. local_pool_name = pool_item[len("local:"):].strip()
  195. if local_pool_name:
  196. local_targets.append(local_pool_name)
  197. elif pool_item.startswith("cloud:"):
  198. cloud_pool_name = pool_item[len("cloud:"):].strip()
  199. if cloud_pool_name:
  200. cloud_targets.append(cloud_pool_name)
  201. else:
  202. raise BizLogicError(
  203. message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
  204. )
  205. if local_targets:
  206. all_local_data = self._load_local_proxies()
  207. nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
  208. nodes_map = {
  209. n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
  210. }
  211. pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
  212. candidate_proxies: List[Dict[str, Any]] = []
  213. for lt in local_targets:
  214. pool_items = pools_dict.get(lt, [])
  215. if pool_items and isinstance(pool_items, list):
  216. for item in pool_items:
  217. if isinstance(item, str):
  218. node_obj = nodes_map.get(item)
  219. if node_obj:
  220. candidate_proxies.append(node_obj)
  221. elif isinstance(item, dict):
  222. candidate_proxies.append(item)
  223. if not candidate_proxies:
  224. raise BizLogicError(
  225. message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
  226. )
  227. # Record node name cache
  228. for node in candidate_proxies:
  229. port = int(node.get("port", 0))
  230. if port and "name" in node:
  231. self._port_to_name[port] = node["name"]
  232. # If no plugin_name is specified, fallback to classic random round-robin
  233. if not plugin_name:
  234. chosen = random.choice(candidate_proxies)
  235. return self._format_proxy_dict(chosen)
  236. # ------------------------------------------------------------------
  237. # Intelligent tiered scheduling (reputation + tiered routing + 5 guardrails)
  238. # ------------------------------------------------------------------
  239. now = time.time()
  240. rep_map = self._get_reputation_map(plugin_name)
  241. with self._lock:
  242. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  243. # Clean up expired quarantine entries
  244. expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
  245. for ip in expired_ips:
  246. quarantine_map.pop(ip, None)
  247. # Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
  248. total_nodes_count = len(candidate_proxies)
  249. quarantined_count = sum(
  250. 1 for node in candidate_proxies
  251. if self._resolve_ip_identifier(node) in quarantine_map
  252. )
  253. # If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
  254. if total_nodes_count > 0:
  255. available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
  256. if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
  257. VSC_WARN("vs_proxy", f"[Proxy Anti-Avalanche] Plugin {plugin_name} available nodes below floor threshold (availability {available_ratio:.1%}). Clearing quarantine list!")
  258. quarantine_map.clear()
  259. # Divide into tiers
  260. trusted_tier: List[Dict[str, Any]] = []
  261. explore_tier: List[Dict[str, Any]] = []
  262. for node in candidate_proxies:
  263. ip_id = self._resolve_ip_identifier(node)
  264. # Check if in quarantine
  265. if ip_id in quarantine_map:
  266. continue
  267. # Check cloud reputation
  268. rep_info = rep_map.get(ip_id)
  269. if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
  270. trusted_tier.append(node)
  271. else:
  272. explore_tier.append(node)
  273. # Tier weighted scheduling: 85% preferred, 15% exploration
  274. chosen = None
  275. if trusted_tier and explore_tier:
  276. if random.random() < 0.85:
  277. chosen = random.choice(trusted_tier)
  278. else:
  279. chosen = random.choice(explore_tier)
  280. elif trusted_tier:
  281. chosen = random.choice(trusted_tier)
  282. elif explore_tier:
  283. chosen = random.choice(explore_tier)
  284. # Layer 5 protection: Hard fallback (if empty after filtering, fallback to full pool random)
  285. if not chosen:
  286. VSC_WARN("vs_proxy", f"[Proxy Hard Fallback] Plugin {plugin_name} has no available nodes after reputation filter. Triggering full random fallback!")
  287. chosen = random.choice(candidate_proxies)
  288. return self._format_proxy_dict(chosen)
  289. if cloud_targets:
  290. return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
  291. raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
  292. def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
  293. """Formats and normalizes the proxy dictionary."""
  294. port = int(chosen.get("port", 80))
  295. egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
  296. proxy_ip = os.getenv("PROXY_HOST") or chosen.get("ip", "127.0.0.1")
  297. return {
  298. "id": chosen.get("id", 0),
  299. "proto": chosen.get("proto", "http"),
  300. "ip": proxy_ip,
  301. "port": port,
  302. "username": chosen.get("username", ""),
  303. "password": chosen.get("password", ""),
  304. "name": chosen.get("name", self._port_to_name.get(port, "")),
  305. "egress_ip": egress_ip,
  306. }
  307. # ==========================================
  308. # Telemetry upstream and real-time feedback
  309. # ==========================================
  310. def mark_proxy_result(
  311. self,
  312. proxy: Dict[str, Any],
  313. plugin_name: str,
  314. action: str,
  315. success: bool,
  316. http_code: int = 200,
  317. error_type: str = "",
  318. latency_ms: int = 0
  319. ):
  320. """
  321. Marks proxy usage outcome and reports asynchronously.
  322. Includes:
  323. - Site-wide outage determination
  324. - Targeted block detection and temporary quarantine
  325. - Half-open revival
  326. - Drift detection
  327. """
  328. if not proxy or not plugin_name:
  329. return
  330. port = int(proxy.get("port", 0))
  331. egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
  332. node_name = proxy.get("name") or self._port_to_name.get(port, "")
  333. # 1. Record into sliding call history
  334. now = time.time()
  335. with self._lock:
  336. history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
  337. history.append({
  338. "ts": now,
  339. "egress_ip": egress_ip,
  340. "success": success,
  341. "http_code": http_code,
  342. "error_type": error_type
  343. })
  344. # 2. Check if currently in site-wide outage
  345. is_outage = self._is_site_outage(plugin_name)
  346. with self._lock:
  347. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  348. if success:
  349. # Success event: If quarantined or half-open, immediately revive and unquarantine
  350. if egress_ip in quarantine_map:
  351. quarantine_map.pop(egress_ip, None)
  352. VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
  353. else:
  354. # Failure event
  355. if is_outage:
  356. VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
  357. else:
  358. # Layer 2: Detailed error classification
  359. is_waf_block = (
  360. "cf-mitigated" in error_type.lower() or
  361. "403201" in error_type or
  362. "403202" in error_type or
  363. "waf blocked" in error_type.lower() or
  364. (http_code == 403 and "mitigated" in error_type.lower())
  365. )
  366. if is_waf_block:
  367. # Targeted block: enter quarantine list (default 1800s)
  368. quarantine_map[egress_ip] = now + 1800
  369. VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF block. Quarantined for 30 minutes!")
  370. # Trigger potential drift detection: invalidate cache to re-probe true public IP
  371. self.invalidate_egress_cache(port)
  372. # 3. Asynchronously report telemetry metrics to cloud (Fail-Silent)
  373. metric_payload = {
  374. "client_id": getattr(configure, "NODE_ID", "local_node"),
  375. "plugin_name": plugin_name,
  376. "local_port": port,
  377. "node_name": node_name,
  378. "egress_ip": egress_ip,
  379. "action": action,
  380. "success": success,
  381. "http_code": http_code,
  382. "error_type": error_type,
  383. "latency_ms": latency_ms,
  384. "timestamp": int(now)
  385. }
  386. def _report_worker():
  387. try:
  388. CloudApi.Instance().report_proxy_metric(metric_payload)
  389. except Exception:
  390. pass
  391. try:
  392. ThreadPool.getInstance().enqueue(_report_worker)
  393. except Exception:
  394. pass