proxy_manager.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451
  1. import os
  2. import time
  3. import json
  4. import yaml
  5. import random
  6. import threading
  7. import collections
  8. import concurrent.futures
  9. from typing import List, Dict, Any, Optional
  10. import requests
  11. import configure
  12. from toolkit.cloud_api import CloudApi
  13. from core_types import BizLogicError
  14. from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
  15. class ProxyManager:
  16. """
  17. Unified proxy scheduling and reputation management hub.
  18. Supports:
  19. - Explicit prefix routing: "local:<pool_name>" / "cloud:<pool_name>"
  20. - Egress IP resolver and cache
  21. - Local channel and cloud reputation intersection weighted scheduling (85% preferred + 15% exploration)
  22. - Five-layer avalanche guardrails (site-wide outage freezing, 20% minimum pool guarantee, half-open revival, hard fallback)
  23. - Asynchronous metric tracking and upstream reporting (fail-silent)
  24. """
  25. _instance = None
  26. def __new__(cls, *args, **kwargs):
  27. if cls._instance is None:
  28. cls._instance = super(ProxyManager, cls).__new__(cls)
  29. cls._instance._init_manager()
  30. return cls._instance
  31. @classmethod
  32. def Instance(cls) -> "ProxyManager":
  33. return cls()
  34. def _init_manager(self):
  35. self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  36. self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
  37. self._lock = threading.RLock()
  38. # Port -> public egress IP cache and node name cache
  39. self._port_to_egress: Dict[int, str] = {}
  40. self._port_to_name: Dict[int, str] = {}
  41. self._probing_ports = set()
  42. # Per-plugin local temporary quarantine blacklist: { plugin_name: { egress_ip: expire_ts } }
  43. self._quarantine: Dict[str, Dict[str, float]] = {}
  44. # Per-plugin cloud reputation cache: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
  45. self._reputation_cache: Dict[str, Dict[str, Any]] = {}
  46. self._fetching_reputation = set()
  47. # Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
  48. self._sliding_history: Dict[str, collections.deque] = {}
  49. def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
  50. if not os.path.exists(self.local_proxies_file):
  51. raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
  52. try:
  53. with open(self.local_proxies_file, "r", encoding="utf-8") as f:
  54. return yaml.safe_load(f) or {}
  55. except Exception as e:
  56. raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
  57. # ==========================================
  58. # Egress IP probe and cache (Egress Resolver)
  59. # ==========================================
  60. def get_cached_egress_ip(self, port: int) -> Optional[str]:
  61. with self._lock:
  62. return self._port_to_egress.get(port)
  63. def invalidate_egress_cache(self, port: int):
  64. """Clears cache when node egress is suspected to have drifted."""
  65. with self._lock:
  66. if port in self._port_to_egress:
  67. old_ip = self._port_to_egress.pop(port, None)
  68. VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
  69. def _async_probe_egress_ip(self, port: int, proto: str = "http"):
  70. """Asynchronously probes the true public egress IP of the port in background, non-blocking."""
  71. with self._lock:
  72. if port in self._port_to_egress or port in self._probing_ports:
  73. return
  74. self._probing_ports.add(port)
  75. def _do_probe():
  76. try:
  77. proxy_url = f"{proto}://127.0.0.1:{port}"
  78. proxies = {"http": proxy_url, "https": proxy_url}
  79. # Try lightweight public IP endpoint
  80. resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
  81. if resp.status_code == 200 and resp.text.strip():
  82. ip = resp.text.strip()
  83. with self._lock:
  84. self._port_to_egress[port] = ip
  85. VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
  86. except Exception:
  87. pass
  88. finally:
  89. with self._lock:
  90. self._probing_ports.discard(port)
  91. t = threading.Thread(target=_do_probe, daemon=True)
  92. t.start()
  93. def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
  94. """Gets the IP identifier of the node (prefers true egress IP; falls back to port anchor if unparsed)."""
  95. port = int(node.get("port", 0))
  96. configured_egress = node.get("egress_ip")
  97. if configured_egress:
  98. return configured_egress
  99. with self._lock:
  100. cached = self._port_to_egress.get(port)
  101. if cached:
  102. return cached
  103. if trigger_probe and port:
  104. proto = node.get("proto", "http")
  105. self._async_probe_egress_ip(port, proto)
  106. return f"port_{port}"
  107. # ==========================================
  108. # Cloud reputation sync and avalanche guardrails
  109. # ==========================================
  110. def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
  111. """Gets cloud reputation mapping (asynchronously refreshed in background without blocking)."""
  112. now = time.time()
  113. with self._lock:
  114. cache_entry = self._reputation_cache.get(plugin_name)
  115. if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
  116. return cache_entry.get("map", {})
  117. if plugin_name in self._fetching_reputation:
  118. return (cache_entry.get("map", {}) if cache_entry else {})
  119. self._fetching_reputation.add(plugin_name)
  120. def _fetch_worker():
  121. try:
  122. rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
  123. with self._lock:
  124. self._reputation_cache[plugin_name] = {
  125. "last_fetch": time.time(),
  126. "map": rep_map or {}
  127. }
  128. except Exception:
  129. pass
  130. finally:
  131. with self._lock:
  132. self._fetching_reputation.discard(plugin_name)
  133. threading.Thread(target=_fetch_worker, daemon=True).start()
  134. return (cache_entry.get("map", {}) if cache_entry else {})
  135. def _is_site_outage(self, plugin_name: str) -> bool:
  136. """
  137. Layer 1 protection: Site-wide outage determination
  138. Within the past 180s, across >= 2 distinct IPs with sample size >= 5:
  139. if failure rate >= 90% or >= 70% 502/503/520~526 errors, judge as site-wide outage.
  140. """
  141. with self._lock:
  142. history = self._sliding_history.get(plugin_name)
  143. if not history or len(history) < 5:
  144. return False
  145. now = time.time()
  146. recent_events = [e for e in history if now - e["ts"] < 180]
  147. if len(recent_events) < 5:
  148. return False
  149. distinct_ips = {e["egress_ip"] for e in recent_events}
  150. if len(distinct_ips) < 2:
  151. return False
  152. failures = [e for e in recent_events if not e["success"]]
  153. fail_rate = len(failures) / len(recent_events)
  154. # Count of severe server-side errors
  155. server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
  156. if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
  157. return True
  158. return False
  159. # ==========================================
  160. # Core scheduling logic (intersection weighting + 5 guardrails)
  161. # ==========================================
  162. def get_next_proxy(
  163. self,
  164. pools: List[str],
  165. proxy_cd: int = 60,
  166. plugin_name: Optional[str] = None
  167. ) -> Dict[str, Any]:
  168. """
  169. Resolves and acquires the next proxy based on pools list.
  170. @param pools: Proxy pool list, e.g. ["local:local_multi_pool"] or ["cloud:proxy-cheap"]
  171. @param proxy_cd: Proxy cooldown time (used for cloud)
  172. @param plugin_name: Plugin name (e.g. 'vfs_plugin'), used for reputation scoring and quarantine
  173. @return dict Standard proxy info (contains id, proto, ip, port, username, password, egress_ip, name)
  174. """
  175. if getattr(configure, "TEST_PROXY", None):
  176. return configure.TEST_PROXY
  177. if not pools:
  178. raise BizLogicError(message="proxy_pool configuration list is empty.")
  179. local_targets = []
  180. cloud_targets = []
  181. for pool_item in pools:
  182. if not isinstance(pool_item, str):
  183. continue
  184. pool_item = pool_item.strip()
  185. if pool_item.startswith("local:"):
  186. local_pool_name = pool_item[len("local:"):].strip()
  187. if local_pool_name:
  188. local_targets.append(local_pool_name)
  189. elif pool_item.startswith("cloud:"):
  190. cloud_pool_name = pool_item[len("cloud:"):].strip()
  191. if cloud_pool_name:
  192. cloud_targets.append(cloud_pool_name)
  193. else:
  194. raise BizLogicError(
  195. message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
  196. )
  197. if local_targets:
  198. all_local_data = self._load_local_proxies()
  199. nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
  200. nodes_map = {
  201. n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
  202. }
  203. pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
  204. candidate_proxies: List[Dict[str, Any]] = []
  205. for lt in local_targets:
  206. pool_items = pools_dict.get(lt, [])
  207. if pool_items and isinstance(pool_items, list):
  208. for item in pool_items:
  209. if isinstance(item, str):
  210. node_obj = nodes_map.get(item)
  211. if node_obj:
  212. candidate_proxies.append(node_obj)
  213. elif isinstance(item, dict):
  214. candidate_proxies.append(item)
  215. if not candidate_proxies:
  216. raise BizLogicError(
  217. message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
  218. )
  219. # Record node name cache
  220. for node in candidate_proxies:
  221. port = int(node.get("port", 0))
  222. if port and "name" in node:
  223. self._port_to_name[port] = node["name"]
  224. # If no plugin_name is specified, fallback to classic random round-robin
  225. if not plugin_name:
  226. chosen = random.choice(candidate_proxies)
  227. return self._format_proxy_dict(chosen)
  228. # ------------------------------------------------------------------
  229. # Intelligent tiered scheduling (reputation + tiered routing + 5 guardrails)
  230. # ------------------------------------------------------------------
  231. now = time.time()
  232. rep_map = self._get_reputation_map(plugin_name)
  233. with self._lock:
  234. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  235. # Clean up expired quarantine entries
  236. expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
  237. for ip in expired_ips:
  238. quarantine_map.pop(ip, None)
  239. # Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
  240. total_nodes_count = len(candidate_proxies)
  241. quarantined_count = sum(
  242. 1 for node in candidate_proxies
  243. if self._resolve_ip_identifier(node) in quarantine_map
  244. )
  245. # If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
  246. if total_nodes_count > 0:
  247. available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
  248. if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
  249. VSC_WARN("vs_proxy", f"[Proxy Anti-Avalanche] Plugin {plugin_name} available nodes below floor threshold (availability {available_ratio:.1%}). Clearing quarantine list!")
  250. quarantine_map.clear()
  251. # Divide into tiers
  252. trusted_tier: List[Dict[str, Any]] = []
  253. explore_tier: List[Dict[str, Any]] = []
  254. for node in candidate_proxies:
  255. ip_id = self._resolve_ip_identifier(node)
  256. # Check if in quarantine
  257. if ip_id in quarantine_map:
  258. continue
  259. # Check cloud reputation
  260. rep_info = rep_map.get(ip_id)
  261. if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
  262. trusted_tier.append(node)
  263. else:
  264. explore_tier.append(node)
  265. # Tier weighted scheduling: 85% preferred, 15% exploration
  266. chosen = None
  267. if trusted_tier and explore_tier:
  268. if random.random() < 0.85:
  269. chosen = random.choice(trusted_tier)
  270. else:
  271. chosen = random.choice(explore_tier)
  272. elif trusted_tier:
  273. chosen = random.choice(trusted_tier)
  274. elif explore_tier:
  275. chosen = random.choice(explore_tier)
  276. # Layer 5 protection: Hard fallback (if empty after filtering, fallback to full pool random)
  277. if not chosen:
  278. VSC_WARN("vs_proxy", f"[Proxy Hard Fallback] Plugin {plugin_name} has no available nodes after reputation filter. Triggering full random fallback!")
  279. chosen = random.choice(candidate_proxies)
  280. return self._format_proxy_dict(chosen)
  281. if cloud_targets:
  282. return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
  283. raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
  284. def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
  285. """Formats and normalizes the proxy dictionary."""
  286. port = int(chosen.get("port", 80))
  287. egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
  288. return {
  289. "id": chosen.get("id", 0),
  290. "proto": chosen.get("proto", "http"),
  291. "ip": chosen.get("ip", "127.0.0.1"),
  292. "port": port,
  293. "username": chosen.get("username", ""),
  294. "password": chosen.get("password", ""),
  295. "name": chosen.get("name", self._port_to_name.get(port, "")),
  296. "egress_ip": egress_ip,
  297. }
  298. # ==========================================
  299. # Telemetry upstream and real-time feedback
  300. # ==========================================
  301. def mark_proxy_result(
  302. self,
  303. proxy: Dict[str, Any],
  304. plugin_name: str,
  305. action: str,
  306. success: bool,
  307. http_code: int = 200,
  308. error_type: str = "",
  309. latency_ms: int = 0
  310. ):
  311. """
  312. Marks proxy usage outcome and reports asynchronously.
  313. Includes:
  314. - Site-wide outage determination
  315. - Targeted block detection and temporary quarantine
  316. - Half-open revival
  317. - Drift detection
  318. """
  319. if not proxy or not plugin_name:
  320. return
  321. port = int(proxy.get("port", 0))
  322. egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
  323. node_name = proxy.get("name") or self._port_to_name.get(port, "")
  324. # 1. Record into sliding call history
  325. now = time.time()
  326. with self._lock:
  327. history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
  328. history.append({
  329. "ts": now,
  330. "egress_ip": egress_ip,
  331. "success": success,
  332. "http_code": http_code,
  333. "error_type": error_type
  334. })
  335. # 2. Check if currently in site-wide outage
  336. is_outage = self._is_site_outage(plugin_name)
  337. with self._lock:
  338. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  339. if success:
  340. # Success event: If quarantined or half-open, immediately revive and unquarantine
  341. if egress_ip in quarantine_map:
  342. quarantine_map.pop(egress_ip, None)
  343. VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
  344. else:
  345. # Failure event
  346. if is_outage:
  347. VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
  348. else:
  349. # Layer 2: Detailed error classification
  350. is_waf_block = (
  351. "cf-mitigated" in error_type.lower() or
  352. "403202" in error_type or
  353. "waf blocked" in error_type.lower() or
  354. (http_code == 403 and "mitigated" in error_type.lower())
  355. )
  356. if is_waf_block:
  357. # Targeted block: enter quarantine list (default 1800s)
  358. quarantine_map[egress_ip] = now + 1800
  359. VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF block. Quarantined for 30 minutes!")
  360. # Trigger potential drift detection: invalidate cache to re-probe true public IP
  361. self.invalidate_egress_cache(port)
  362. # 3. Asynchronously report telemetry metrics to cloud (Fail-Silent)
  363. metric_payload = {
  364. "client_id": getattr(configure, "NODE_ID", "local_node"),
  365. "plugin_name": plugin_name,
  366. "local_port": port,
  367. "node_name": node_name,
  368. "egress_ip": egress_ip,
  369. "action": action,
  370. "success": success,
  371. "http_code": http_code,
  372. "error_type": error_type,
  373. "latency_ms": latency_ms,
  374. "timestamp": int(now)
  375. }
  376. def _report_worker():
  377. try:
  378. CloudApi.Instance().report_proxy_metric(metric_payload)
  379. except Exception:
  380. pass
  381. t = threading.Thread(target=_report_worker, daemon=True)
  382. t.start()