proxy_manager.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511
  1. import os
  2. import time
  3. import json
  4. import yaml
  5. import random
  6. import threading
  7. import collections
  8. import concurrent.futures
  9. from typing import List, Dict, Any, Optional, Tuple
  10. import requests
  11. import configure
  12. from toolkit.cloud_api import CloudApi
  13. from toolkit.task_executor import submit_background
  14. from core_types import BizLogicError
  15. from logger_setup import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
  16. class ProxyManager:
  17. """
  18. Unified proxy scheduling and reputation management hub.
  19. Supports:
  20. - Explicit prefix routing: "local:<pool_name>" / "cloud:<pool_name>"
  21. - Egress IP resolver and cache
  22. - Local channel and cloud reputation intersection weighted scheduling (85% preferred + 15% exploration)
  23. - Five-layer avalanche guardrails (site-wide outage freezing, 20% minimum pool guarantee, half-open revival, hard fallback)
  24. - Asynchronous metric tracking and upstream reporting (fail-silent)
  25. """
  26. _instance = None
  27. def __new__(cls, *args, **kwargs):
  28. if cls._instance is None:
  29. cls._instance = super(ProxyManager, cls).__new__(cls)
  30. cls._instance._init_manager()
  31. return cls._instance
  32. @classmethod
  33. def Instance(cls) -> "ProxyManager":
  34. return cls()
  35. def _init_manager(self):
  36. self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  37. self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
  38. self._lock = threading.RLock()
  39. # Port -> public egress IP cache and node name cache
  40. self._port_to_egress: Dict[int, str] = {}
  41. self._port_to_name: Dict[int, str] = {}
  42. self._probing_ports = set()
  43. # Per-plugin local temporary quarantine blacklist: { plugin_name: { egress_ip: expire_ts } }
  44. self._quarantine: Dict[str, Dict[str, float]] = {}
  45. # Per-plugin cloud reputation cache: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
  46. self._reputation_cache: Dict[str, Dict[str, Any]] = {}
  47. self._fetching_reputation = set()
  48. # Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
  49. self._sliding_history: Dict[str, collections.deque] = {}
  50. # Per-plugin node cooldown / last-used tracking: { plugin_name: { identifier: expire_ts } }
  51. self._node_cooldown: Dict[str, Dict[str, float]] = {}
  52. def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
  53. if not os.path.exists(self.local_proxies_file):
  54. raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
  55. try:
  56. with open(self.local_proxies_file, "r", encoding="utf-8") as f:
  57. return yaml.safe_load(f) or {}
  58. except Exception as e:
  59. raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
  60. # ==========================================
  61. # Egress IP probe and cache (Egress Resolver)
  62. # ==========================================
  63. def get_cached_egress_ip(self, port: int) -> Optional[str]:
  64. with self._lock:
  65. return self._port_to_egress.get(port)
  66. def invalidate_egress_cache(self, port: int):
  67. """Clears cache when node egress is suspected to have drifted."""
  68. with self._lock:
  69. if port in self._port_to_egress:
  70. old_ip = self._port_to_egress.pop(port, None)
  71. VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
  72. def _async_probe_egress_ip(self, port: int, proto: str = "http"):
  73. """Asynchronously probes the true public egress IP of the port in background, non-blocking."""
  74. with self._lock:
  75. if port in self._port_to_egress or port in self._probing_ports:
  76. return
  77. self._probing_ports.add(port)
  78. def _do_probe():
  79. try:
  80. host = os.getenv("PROXY_HOST") or "127.0.0.1"
  81. proxy_url = f"{proto}://{host}:{port}"
  82. proxies = {"http": proxy_url, "https": proxy_url}
  83. # Try lightweight public IP endpoint
  84. resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
  85. if resp.status_code == 200 and resp.text.strip():
  86. ip = resp.text.strip()
  87. with self._lock:
  88. self._port_to_egress[port] = ip
  89. VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
  90. except Exception:
  91. pass
  92. finally:
  93. with self._lock:
  94. self._probing_ports.discard(port)
  95. if not submit_background(_do_probe):
  96. t = threading.Thread(target=_do_probe, daemon=True)
  97. t.start()
  98. def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
  99. """Gets the IP identifier of the node (prefers true egress IP; falls back to port anchor if unparsed)."""
  100. port = int(node.get("port", 0))
  101. configured_egress = node.get("egress_ip")
  102. if configured_egress:
  103. return configured_egress
  104. with self._lock:
  105. cached = self._port_to_egress.get(port)
  106. if cached:
  107. return cached
  108. if trigger_probe and port:
  109. proto = node.get("proto", "http")
  110. self._async_probe_egress_ip(port, proto)
  111. return f"port_{port}"
  112. # ==========================================
  113. # Cloud reputation sync and avalanche guardrails
  114. # ==========================================
  115. def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
  116. """Gets cloud reputation mapping (asynchronously refreshed in background without blocking)."""
  117. now = time.time()
  118. with self._lock:
  119. cache_entry = self._reputation_cache.get(plugin_name)
  120. if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
  121. return cache_entry.get("map", {})
  122. if plugin_name in self._fetching_reputation:
  123. return (cache_entry.get("map", {}) if cache_entry else {})
  124. self._fetching_reputation.add(plugin_name)
  125. def _fetch_worker():
  126. try:
  127. rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
  128. with self._lock:
  129. self._reputation_cache[plugin_name] = {
  130. "last_fetch": time.time(),
  131. "map": rep_map or {}
  132. }
  133. except Exception:
  134. pass
  135. finally:
  136. with self._lock:
  137. self._fetching_reputation.discard(plugin_name)
  138. if not submit_background(_fetch_worker):
  139. threading.Thread(target=_fetch_worker, daemon=True).start()
  140. return (cache_entry.get("map", {}) if cache_entry else {})
  141. def _is_site_outage(self, plugin_name: str) -> bool:
  142. """
  143. Layer 1 protection: Site-wide outage determination
  144. Within the past 180s, across >= 2 distinct IPs with sample size >= 5:
  145. if failure rate >= 90% or >= 70% 502/503/520~526 errors, judge as site-wide outage.
  146. """
  147. with self._lock:
  148. history = self._sliding_history.get(plugin_name)
  149. if not history or len(history) < 5:
  150. return False
  151. now = time.time()
  152. recent_events = [e for e in history if now - e["ts"] < 180]
  153. if len(recent_events) < 5:
  154. return False
  155. distinct_ips = {e["egress_ip"] for e in recent_events}
  156. if len(distinct_ips) < 2:
  157. return False
  158. failures = [e for e in recent_events if not e["success"]]
  159. fail_rate = len(failures) / len(recent_events)
  160. # Count of severe server-side errors
  161. server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
  162. if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
  163. return True
  164. return False
  165. # ==========================================
  166. # Core scheduling logic (intersection weighting + 5 guardrails)
  167. # ==========================================
  168. def get_next_proxy(
  169. self,
  170. pools: List[str],
  171. proxy_cd: int = 60,
  172. plugin_name: Optional[str] = None
  173. ) -> Dict[str, Any]:
  174. """
  175. Resolves and acquires the next proxy based on pools list.
  176. @param pools: Proxy pool list, e.g. ["local:local_multi_pool"] or ["cloud:proxy-cheap"]
  177. @param proxy_cd: Proxy cooldown time (used for cloud)
  178. @param plugin_name: Plugin name (e.g. 'vfs_plugin'), used for reputation scoring and quarantine
  179. @return dict Standard proxy info (contains id, proto, ip, port, username, password, egress_ip, name)
  180. """
  181. if getattr(configure, "TEST_PROXY", None):
  182. return configure.TEST_PROXY
  183. if not pools:
  184. raise BizLogicError(message="proxy_pool configuration list is empty.")
  185. local_targets = []
  186. cloud_targets = []
  187. for pool_item in pools:
  188. if not isinstance(pool_item, str):
  189. continue
  190. pool_item = pool_item.strip()
  191. if pool_item.startswith("local:"):
  192. local_pool_name = pool_item[len("local:"):].strip()
  193. if local_pool_name:
  194. local_targets.append(local_pool_name)
  195. elif pool_item.startswith("cloud:"):
  196. cloud_pool_name = pool_item[len("cloud:"):].strip()
  197. if cloud_pool_name:
  198. cloud_targets.append(cloud_pool_name)
  199. else:
  200. raise BizLogicError(
  201. message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
  202. )
  203. if local_targets:
  204. all_local_data = self._load_local_proxies()
  205. nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
  206. nodes_map = {
  207. n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
  208. }
  209. pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
  210. candidate_proxies: List[Dict[str, Any]] = []
  211. for lt in local_targets:
  212. pool_items = pools_dict.get(lt, [])
  213. if pool_items and isinstance(pool_items, list):
  214. for item in pool_items:
  215. if isinstance(item, str):
  216. node_obj = nodes_map.get(item)
  217. if node_obj:
  218. candidate_proxies.append(node_obj)
  219. elif isinstance(item, dict):
  220. candidate_proxies.append(item)
  221. if not candidate_proxies:
  222. raise BizLogicError(
  223. message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
  224. )
  225. # Record node name cache
  226. for node in candidate_proxies:
  227. port = int(node.get("port", 0))
  228. if port and "name" in node:
  229. self._port_to_name[port] = node["name"]
  230. # If no plugin_name is specified, fallback to classic random round-robin
  231. if not plugin_name:
  232. chosen = random.choice(candidate_proxies)
  233. return self._format_proxy_dict(chosen)
  234. # ------------------------------------------------------------------
  235. # Intelligent tiered scheduling (reputation + tiered routing + 5 guardrails)
  236. # ------------------------------------------------------------------
  237. now = time.time()
  238. rep_map = self._get_reputation_map(plugin_name)
  239. effective_cd = max(float(proxy_cd or 60.0), 30.0)
  240. with self._lock:
  241. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  242. cooldown_map = self._node_cooldown.setdefault(plugin_name, {})
  243. # Clean up expired quarantine entries
  244. expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
  245. for ip in expired_ips:
  246. quarantine_map.pop(ip, None)
  247. # Clean up expired cooldown entries
  248. expired_cd = [k for k, exp in cooldown_map.items() if now > exp]
  249. for k in expired_cd:
  250. cooldown_map.pop(k, None)
  251. # Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
  252. total_nodes_count = len(candidate_proxies)
  253. quarantined_count = sum(
  254. 1 for node in candidate_proxies
  255. if self._resolve_ip_identifier(node) in quarantine_map or f"port_{int(node.get('port', 0))}" in quarantine_map
  256. )
  257. # If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
  258. if total_nodes_count > 0:
  259. available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
  260. if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
  261. VSC_WARN("vs_proxy", f"[Proxy Anti-Avalanche] Plugin {plugin_name} available nodes below floor threshold (availability {available_ratio:.1%}). Clearing quarantine list!")
  262. quarantine_map.clear()
  263. # Divide into tiers
  264. trusted_tier: List[Dict[str, Any]] = []
  265. explore_tier: List[Dict[str, Any]] = []
  266. for node in candidate_proxies:
  267. port = int(node.get("port", 0))
  268. ip_id = self._resolve_ip_identifier(node)
  269. # Check if in quarantine (check both ip_id and port)
  270. if ip_id in quarantine_map or (port and f"port_{port}" in quarantine_map):
  271. continue
  272. # Check cloud reputation (check both ip_id and port)
  273. rep_info = rep_map.get(ip_id)
  274. if not rep_info and port:
  275. rep_info = rep_map.get(f"port_{port}")
  276. if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
  277. trusted_tier.append(node)
  278. else:
  279. explore_tier.append(node)
  280. # Proportional dynamic trust factor:
  281. # Trusted tier ratio dynamically scales with the number of trusted nodes (up to max 0.85).
  282. # When only 1 node is trusted, it gets at most 20% of traffic, ensuring 80% explores other nodes to avoid single-node monopolization.
  283. trusted_ratio = min(0.85, 0.20 * len(trusted_tier)) if (trusted_tier and explore_tier) else (1.0 if trusted_tier else 0.0)
  284. target_tier: List[Dict[str, Any]] = []
  285. if trusted_tier and explore_tier:
  286. target_tier = trusted_tier if (random.random() < trusted_ratio) else explore_tier
  287. elif trusted_tier:
  288. target_tier = trusted_tier
  289. elif explore_tier:
  290. target_tier = explore_tier
  291. else:
  292. target_tier = candidate_proxies
  293. # Anti-Affinity & Local Cooldown Selection:
  294. # Prefer non-cooling nodes. If all nodes in tier are cooling, choose the least-recently used.
  295. def _is_cooling(node: Dict[str, Any]) -> Tuple[bool, float]:
  296. p = int(node.get("port", 0))
  297. iid = self._resolve_ip_identifier(node)
  298. cd_exp = max(cooldown_map.get(iid, 0), cooldown_map.get(f"port_{p}", 0) if p else 0)
  299. return (cd_exp > now, cd_exp)
  300. non_cooling = [n for n in target_tier if not _is_cooling(n)[0]]
  301. if non_cooling:
  302. chosen = random.choice(non_cooling)
  303. elif target_tier:
  304. # All candidates in tier are cooling, pick the one that expires earliest
  305. chosen = min(target_tier, key=lambda n: _is_cooling(n)[1])
  306. else:
  307. chosen = random.choice(candidate_proxies)
  308. # Record cooldown on the chosen node
  309. chosen_port = int(chosen.get("port", 0))
  310. chosen_id = self._resolve_ip_identifier(chosen)
  311. cooldown_until = now + effective_cd
  312. with self._lock:
  313. cooldown_map[chosen_id] = cooldown_until
  314. if chosen_port:
  315. cooldown_map[f"port_{chosen_port}"] = cooldown_until
  316. return self._format_proxy_dict(chosen)
  317. if cloud_targets:
  318. return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
  319. raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
  320. def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
  321. """Formats and normalizes the proxy dictionary."""
  322. port = int(chosen.get("port", 80))
  323. egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
  324. proxy_ip = os.getenv("PROXY_HOST") or chosen.get("ip", "127.0.0.1")
  325. return {
  326. "id": chosen.get("id", 0),
  327. "proto": chosen.get("proto", "http"),
  328. "ip": proxy_ip,
  329. "port": port,
  330. "username": chosen.get("username", ""),
  331. "password": chosen.get("password", ""),
  332. "name": chosen.get("name", self._port_to_name.get(port, "")),
  333. "egress_ip": egress_ip,
  334. }
  335. # ==========================================
  336. # Telemetry upstream and real-time feedback
  337. # ==========================================
  338. def mark_proxy_result(
  339. self,
  340. proxy: Dict[str, Any],
  341. plugin_name: str,
  342. action: str,
  343. success: bool,
  344. http_code: int = 200,
  345. error_type: str = "",
  346. latency_ms: int = 0
  347. ):
  348. """
  349. Marks proxy usage outcome and reports asynchronously.
  350. Includes:
  351. - Site-wide outage determination
  352. - Targeted block detection and temporary quarantine
  353. - Half-open revival
  354. - Drift detection
  355. """
  356. if not proxy or not plugin_name:
  357. return
  358. port = int(proxy.get("port", 0))
  359. egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
  360. node_name = proxy.get("name") or self._port_to_name.get(port, "")
  361. # 1. Record into sliding call history
  362. now = time.time()
  363. with self._lock:
  364. history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
  365. history.append({
  366. "ts": now,
  367. "egress_ip": egress_ip,
  368. "success": success,
  369. "http_code": http_code,
  370. "error_type": error_type
  371. })
  372. # 2. Check if currently in site-wide outage
  373. is_outage = self._is_site_outage(plugin_name)
  374. with self._lock:
  375. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  376. cooldown_map = self._node_cooldown.setdefault(plugin_name, {})
  377. if success:
  378. # Success event: If quarantined or half-open, immediately revive and unquarantine
  379. revived = False
  380. if egress_ip in quarantine_map:
  381. quarantine_map.pop(egress_ip, None)
  382. revived = True
  383. if port and f"port_{port}" in quarantine_map:
  384. quarantine_map.pop(f"port_{port}", None)
  385. revived = True
  386. if revived:
  387. VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
  388. else:
  389. # Failure event: Always apply transient failure cooldown (180s) to prevent immediate re-hammering
  390. fail_cd_until = now + 180.0
  391. cooldown_map[egress_ip] = max(cooldown_map.get(egress_ip, 0), fail_cd_until)
  392. if port:
  393. cooldown_map[f"port_{port}"] = max(cooldown_map.get(f"port_{port}", 0), fail_cd_until)
  394. if is_outage:
  395. VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
  396. else:
  397. # Layer 2: Detailed error classification
  398. is_waf_block = (
  399. "cf-mitigated" in error_type.lower() or
  400. "403201" in error_type or
  401. "403202" in error_type or
  402. "waf blocked" in error_type.lower() or
  403. "turnstile" in error_type.lower() or
  404. "challenge" in error_type.lower() or
  405. "cloudflare" in error_type.lower() or
  406. (http_code == 403 and "mitigated" in error_type.lower())
  407. )
  408. if is_waf_block:
  409. # Targeted block: enter quarantine list (default 1800s)
  410. quarantine_map[egress_ip] = now + 1800
  411. if port:
  412. quarantine_map[f"port_{port}"] = now + 1800
  413. VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF/Turnstile block. Quarantined for 30 minutes!")
  414. # Trigger potential drift detection: invalidate cache to re-probe true public IP
  415. self.invalidate_egress_cache(port)
  416. # 3. Asynchronously report telemetry metrics to cloud (Fail-Silent)
  417. metric_payload = {
  418. "client_id": getattr(configure, "NODE_ID", "local_node"),
  419. "plugin_name": plugin_name,
  420. "local_port": port,
  421. "node_name": node_name,
  422. "egress_ip": egress_ip,
  423. "action": action,
  424. "success": success,
  425. "http_code": http_code,
  426. "error_type": error_type,
  427. "latency_ms": latency_ms,
  428. "timestamp": int(now)
  429. }
  430. def _report_worker():
  431. try:
  432. CloudApi.Instance().report_proxy_metric(metric_payload)
  433. except Exception:
  434. pass
  435. submit_background(_report_worker)