proxy_manager.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453
  1. import os
  2. import time
  3. import json
  4. import yaml
  5. import random
  6. import threading
  7. import collections
  8. import concurrent.futures
  9. from typing import List, Dict, Any, Optional
  10. import requests
  11. import configure
  12. from toolkit.cloud_api import CloudApi
  13. from core_types import BizLogicError
  14. from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
  15. class ProxyManager:
  16. """
  17. Unified proxy scheduling and reputation management hub.
  18. Supports:
  19. - Explicit prefix routing: "local:<pool_name>" / "cloud:<pool_name>"
  20. - Egress IP resolver and cache
  21. - Local channel and cloud reputation intersection weighted scheduling (85% preferred + 15% exploration)
  22. - Five-layer avalanche guardrails (site-wide outage freezing, 20% minimum pool guarantee, half-open revival, hard fallback)
  23. - Asynchronous metric tracking and upstream reporting (fail-silent)
  24. """
  25. _instance = None
  26. def __new__(cls, *args, **kwargs):
  27. if cls._instance is None:
  28. cls._instance = super(ProxyManager, cls).__new__(cls)
  29. cls._instance._init_manager()
  30. return cls._instance
  31. @classmethod
  32. def Instance(cls) -> "ProxyManager":
  33. return cls()
  34. def _init_manager(self):
  35. self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  36. self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
  37. self._lock = threading.RLock()
  38. # Port -> public egress IP cache and node name cache
  39. self._port_to_egress: Dict[int, str] = {}
  40. self._port_to_name: Dict[int, str] = {}
  41. self._probing_ports = set()
  42. # Per-plugin local temporary quarantine blacklist: { plugin_name: { egress_ip: expire_ts } }
  43. self._quarantine: Dict[str, Dict[str, float]] = {}
  44. # Per-plugin cloud reputation cache: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
  45. self._reputation_cache: Dict[str, Dict[str, Any]] = {}
  46. self._fetching_reputation = set()
  47. # Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
  48. self._sliding_history: Dict[str, collections.deque] = {}
  49. def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
  50. if not os.path.exists(self.local_proxies_file):
  51. raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
  52. try:
  53. with open(self.local_proxies_file, "r", encoding="utf-8") as f:
  54. return yaml.safe_load(f) or {}
  55. except Exception as e:
  56. raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
  57. # ==========================================
  58. # Egress IP probe and cache (Egress Resolver)
  59. # ==========================================
  60. def get_cached_egress_ip(self, port: int) -> Optional[str]:
  61. with self._lock:
  62. return self._port_to_egress.get(port)
  63. def invalidate_egress_cache(self, port: int):
  64. """Clears cache when node egress is suspected to have drifted."""
  65. with self._lock:
  66. if port in self._port_to_egress:
  67. old_ip = self._port_to_egress.pop(port, None)
  68. VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
  69. def _async_probe_egress_ip(self, port: int, proto: str = "http"):
  70. """Asynchronously probes the true public egress IP of the port in background, non-blocking."""
  71. with self._lock:
  72. if port in self._port_to_egress or port in self._probing_ports:
  73. return
  74. self._probing_ports.add(port)
  75. def _do_probe():
  76. try:
  77. host = os.getenv("PROXY_HOST") or "127.0.0.1"
  78. proxy_url = f"{proto}://{host}:{port}"
  79. proxies = {"http": proxy_url, "https": proxy_url}
  80. # Try lightweight public IP endpoint
  81. resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
  82. if resp.status_code == 200 and resp.text.strip():
  83. ip = resp.text.strip()
  84. with self._lock:
  85. self._port_to_egress[port] = ip
  86. VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
  87. except Exception:
  88. pass
  89. finally:
  90. with self._lock:
  91. self._probing_ports.discard(port)
  92. t = threading.Thread(target=_do_probe, daemon=True)
  93. t.start()
  94. def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
  95. """Gets the IP identifier of the node (prefers true egress IP; falls back to port anchor if unparsed)."""
  96. port = int(node.get("port", 0))
  97. configured_egress = node.get("egress_ip")
  98. if configured_egress:
  99. return configured_egress
  100. with self._lock:
  101. cached = self._port_to_egress.get(port)
  102. if cached:
  103. return cached
  104. if trigger_probe and port:
  105. proto = node.get("proto", "http")
  106. self._async_probe_egress_ip(port, proto)
  107. return f"port_{port}"
  108. # ==========================================
  109. # Cloud reputation sync and avalanche guardrails
  110. # ==========================================
  111. def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
  112. """Gets cloud reputation mapping (asynchronously refreshed in background without blocking)."""
  113. now = time.time()
  114. with self._lock:
  115. cache_entry = self._reputation_cache.get(plugin_name)
  116. if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
  117. return cache_entry.get("map", {})
  118. if plugin_name in self._fetching_reputation:
  119. return (cache_entry.get("map", {}) if cache_entry else {})
  120. self._fetching_reputation.add(plugin_name)
  121. def _fetch_worker():
  122. try:
  123. rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
  124. with self._lock:
  125. self._reputation_cache[plugin_name] = {
  126. "last_fetch": time.time(),
  127. "map": rep_map or {}
  128. }
  129. except Exception:
  130. pass
  131. finally:
  132. with self._lock:
  133. self._fetching_reputation.discard(plugin_name)
  134. threading.Thread(target=_fetch_worker, daemon=True).start()
  135. return (cache_entry.get("map", {}) if cache_entry else {})
  136. def _is_site_outage(self, plugin_name: str) -> bool:
  137. """
  138. Layer 1 protection: Site-wide outage determination
  139. Within the past 180s, across >= 2 distinct IPs with sample size >= 5:
  140. if failure rate >= 90% or >= 70% 502/503/520~526 errors, judge as site-wide outage.
  141. """
  142. with self._lock:
  143. history = self._sliding_history.get(plugin_name)
  144. if not history or len(history) < 5:
  145. return False
  146. now = time.time()
  147. recent_events = [e for e in history if now - e["ts"] < 180]
  148. if len(recent_events) < 5:
  149. return False
  150. distinct_ips = {e["egress_ip"] for e in recent_events}
  151. if len(distinct_ips) < 2:
  152. return False
  153. failures = [e for e in recent_events if not e["success"]]
  154. fail_rate = len(failures) / len(recent_events)
  155. # Count of severe server-side errors
  156. server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
  157. if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
  158. return True
  159. return False
  160. # ==========================================
  161. # Core scheduling logic (intersection weighting + 5 guardrails)
  162. # ==========================================
  163. def get_next_proxy(
  164. self,
  165. pools: List[str],
  166. proxy_cd: int = 60,
  167. plugin_name: Optional[str] = None
  168. ) -> Dict[str, Any]:
  169. """
  170. Resolves and acquires the next proxy based on pools list.
  171. @param pools: Proxy pool list, e.g. ["local:local_multi_pool"] or ["cloud:proxy-cheap"]
  172. @param proxy_cd: Proxy cooldown time (used for cloud)
  173. @param plugin_name: Plugin name (e.g. 'vfs_plugin'), used for reputation scoring and quarantine
  174. @return dict Standard proxy info (contains id, proto, ip, port, username, password, egress_ip, name)
  175. """
  176. if getattr(configure, "TEST_PROXY", None):
  177. return configure.TEST_PROXY
  178. if not pools:
  179. raise BizLogicError(message="proxy_pool configuration list is empty.")
  180. local_targets = []
  181. cloud_targets = []
  182. for pool_item in pools:
  183. if not isinstance(pool_item, str):
  184. continue
  185. pool_item = pool_item.strip()
  186. if pool_item.startswith("local:"):
  187. local_pool_name = pool_item[len("local:"):].strip()
  188. if local_pool_name:
  189. local_targets.append(local_pool_name)
  190. elif pool_item.startswith("cloud:"):
  191. cloud_pool_name = pool_item[len("cloud:"):].strip()
  192. if cloud_pool_name:
  193. cloud_targets.append(cloud_pool_name)
  194. else:
  195. raise BizLogicError(
  196. message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
  197. )
  198. if local_targets:
  199. all_local_data = self._load_local_proxies()
  200. nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
  201. nodes_map = {
  202. n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
  203. }
  204. pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
  205. candidate_proxies: List[Dict[str, Any]] = []
  206. for lt in local_targets:
  207. pool_items = pools_dict.get(lt, [])
  208. if pool_items and isinstance(pool_items, list):
  209. for item in pool_items:
  210. if isinstance(item, str):
  211. node_obj = nodes_map.get(item)
  212. if node_obj:
  213. candidate_proxies.append(node_obj)
  214. elif isinstance(item, dict):
  215. candidate_proxies.append(item)
  216. if not candidate_proxies:
  217. raise BizLogicError(
  218. message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
  219. )
  220. # Record node name cache
  221. for node in candidate_proxies:
  222. port = int(node.get("port", 0))
  223. if port and "name" in node:
  224. self._port_to_name[port] = node["name"]
  225. # If no plugin_name is specified, fallback to classic random round-robin
  226. if not plugin_name:
  227. chosen = random.choice(candidate_proxies)
  228. return self._format_proxy_dict(chosen)
  229. # ------------------------------------------------------------------
  230. # Intelligent tiered scheduling (reputation + tiered routing + 5 guardrails)
  231. # ------------------------------------------------------------------
  232. now = time.time()
  233. rep_map = self._get_reputation_map(plugin_name)
  234. with self._lock:
  235. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  236. # Clean up expired quarantine entries
  237. expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
  238. for ip in expired_ips:
  239. quarantine_map.pop(ip, None)
  240. # Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
  241. total_nodes_count = len(candidate_proxies)
  242. quarantined_count = sum(
  243. 1 for node in candidate_proxies
  244. if self._resolve_ip_identifier(node) in quarantine_map
  245. )
  246. # If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
  247. if total_nodes_count > 0:
  248. available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
  249. if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
  250. VSC_WARN("vs_proxy", f"[Proxy Anti-Avalanche] Plugin {plugin_name} available nodes below floor threshold (availability {available_ratio:.1%}). Clearing quarantine list!")
  251. quarantine_map.clear()
  252. # Divide into tiers
  253. trusted_tier: List[Dict[str, Any]] = []
  254. explore_tier: List[Dict[str, Any]] = []
  255. for node in candidate_proxies:
  256. ip_id = self._resolve_ip_identifier(node)
  257. # Check if in quarantine
  258. if ip_id in quarantine_map:
  259. continue
  260. # Check cloud reputation
  261. rep_info = rep_map.get(ip_id)
  262. if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
  263. trusted_tier.append(node)
  264. else:
  265. explore_tier.append(node)
  266. # Tier weighted scheduling: 85% preferred, 15% exploration
  267. chosen = None
  268. if trusted_tier and explore_tier:
  269. if random.random() < 0.85:
  270. chosen = random.choice(trusted_tier)
  271. else:
  272. chosen = random.choice(explore_tier)
  273. elif trusted_tier:
  274. chosen = random.choice(trusted_tier)
  275. elif explore_tier:
  276. chosen = random.choice(explore_tier)
  277. # Layer 5 protection: Hard fallback (if empty after filtering, fallback to full pool random)
  278. if not chosen:
  279. VSC_WARN("vs_proxy", f"[Proxy Hard Fallback] Plugin {plugin_name} has no available nodes after reputation filter. Triggering full random fallback!")
  280. chosen = random.choice(candidate_proxies)
  281. return self._format_proxy_dict(chosen)
  282. if cloud_targets:
  283. return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
  284. raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
  285. def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
  286. """Formats and normalizes the proxy dictionary."""
  287. port = int(chosen.get("port", 80))
  288. egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
  289. proxy_ip = os.getenv("PROXY_HOST") or chosen.get("ip", "127.0.0.1")
  290. return {
  291. "id": chosen.get("id", 0),
  292. "proto": chosen.get("proto", "http"),
  293. "ip": proxy_ip,
  294. "port": port,
  295. "username": chosen.get("username", ""),
  296. "password": chosen.get("password", ""),
  297. "name": chosen.get("name", self._port_to_name.get(port, "")),
  298. "egress_ip": egress_ip,
  299. }
  300. # ==========================================
  301. # Telemetry upstream and real-time feedback
  302. # ==========================================
  303. def mark_proxy_result(
  304. self,
  305. proxy: Dict[str, Any],
  306. plugin_name: str,
  307. action: str,
  308. success: bool,
  309. http_code: int = 200,
  310. error_type: str = "",
  311. latency_ms: int = 0
  312. ):
  313. """
  314. Marks proxy usage outcome and reports asynchronously.
  315. Includes:
  316. - Site-wide outage determination
  317. - Targeted block detection and temporary quarantine
  318. - Half-open revival
  319. - Drift detection
  320. """
  321. if not proxy or not plugin_name:
  322. return
  323. port = int(proxy.get("port", 0))
  324. egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
  325. node_name = proxy.get("name") or self._port_to_name.get(port, "")
  326. # 1. Record into sliding call history
  327. now = time.time()
  328. with self._lock:
  329. history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
  330. history.append({
  331. "ts": now,
  332. "egress_ip": egress_ip,
  333. "success": success,
  334. "http_code": http_code,
  335. "error_type": error_type
  336. })
  337. # 2. Check if currently in site-wide outage
  338. is_outage = self._is_site_outage(plugin_name)
  339. with self._lock:
  340. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  341. if success:
  342. # Success event: If quarantined or half-open, immediately revive and unquarantine
  343. if egress_ip in quarantine_map:
  344. quarantine_map.pop(egress_ip, None)
  345. VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
  346. else:
  347. # Failure event
  348. if is_outage:
  349. VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
  350. else:
  351. # Layer 2: Detailed error classification
  352. is_waf_block = (
  353. "cf-mitigated" in error_type.lower() or
  354. "403202" in error_type or
  355. "waf blocked" in error_type.lower() or
  356. (http_code == 403 and "mitigated" in error_type.lower())
  357. )
  358. if is_waf_block:
  359. # Targeted block: enter quarantine list (default 1800s)
  360. quarantine_map[egress_ip] = now + 1800
  361. VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF block. Quarantined for 30 minutes!")
  362. # Trigger potential drift detection: invalidate cache to re-probe true public IP
  363. self.invalidate_egress_cache(port)
  364. # 3. Asynchronously report telemetry metrics to cloud (Fail-Silent)
  365. metric_payload = {
  366. "client_id": getattr(configure, "NODE_ID", "local_node"),
  367. "plugin_name": plugin_name,
  368. "local_port": port,
  369. "node_name": node_name,
  370. "egress_ip": egress_ip,
  371. "action": action,
  372. "success": success,
  373. "http_code": http_code,
  374. "error_type": error_type,
  375. "latency_ms": latency_ms,
  376. "timestamp": int(now)
  377. }
  378. def _report_worker():
  379. try:
  380. CloudApi.Instance().report_proxy_metric(metric_payload)
  381. except Exception:
  382. pass
  383. t = threading.Thread(target=_report_worker, daemon=True)
  384. t.start()