proxy_manager.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450
  1. import os
  2. import time
  3. import json
  4. import yaml
  5. import random
  6. import threading
  7. import collections
  8. import concurrent.futures
  9. from typing import List, Dict, Any, Optional
  10. import requests
  11. import configure
  12. from toolkit.cloud_api import CloudApi
  13. from core_types import BizLogicError
  14. from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
  15. class ProxyManager:
  16. """
  17. 统一代理调度与信誉管理中心。
  18. 支持:
  19. - 显式前缀区分:"local:<pool_name>" / "cloud:<pool_name>"
  20. - 出口 IP 探针与缓存机制 (Egress Resolver & Cache)
  21. - 本地通道与云端信誉库求交加权调度 (85% 优选 + 15% 探索)
  22. - 五层防雪崩护栏 (全站故障冻结、20% 最低保底池、半开自愈、底层硬兜底)
  23. - 异步指标埋点与上报 (Fail-Silent)
  24. """
  25. _instance = None
  26. def __new__(cls, *args, **kwargs):
  27. if cls._instance is None:
  28. cls._instance = super(ProxyManager, cls).__new__(cls)
  29. cls._instance._init_manager()
  30. return cls._instance
  31. @classmethod
  32. def Instance(cls) -> "ProxyManager":
  33. return cls()
  34. def _init_manager(self):
  35. self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  36. self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
  37. self._lock = threading.RLock()
  38. # 端口 -> 公网出口 IP 缓存与节点名称缓存
  39. self._port_to_egress: Dict[int, str] = {}
  40. self._port_to_name: Dict[int, str] = {}
  41. self._probing_ports = set()
  42. # 插件维度本地临时隔离黑名单: { plugin_name: { egress_ip: expire_ts } }
  43. self._quarantine: Dict[str, Dict[str, float]] = {}
  44. # 插件维度云端信誉缓存: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
  45. self._reputation_cache: Dict[str, Dict[str, Any]] = {}
  46. self._fetching_reputation = set()
  47. # 插件维度滑动调用历史 (用于全站故障判定): { plugin_name: collections.deque(maxlen=30) }
  48. self._sliding_history: Dict[str, collections.deque] = {}
  49. def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
  50. if not os.path.exists(self.local_proxies_file):
  51. raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
  52. try:
  53. with open(self.local_proxies_file, "r", encoding="utf-8") as f:
  54. return yaml.safe_load(f) or {}
  55. except Exception as e:
  56. raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
  57. # ==========================================
  58. # 出口 IP 探针与缓存 (Egress Resolver)
  59. # ==========================================
  60. def get_cached_egress_ip(self, port: int) -> Optional[str]:
  61. with self._lock:
  62. return self._port_to_egress.get(port)
  63. def invalidate_egress_cache(self, port: int):
  64. """当节点疑似漂移时清除缓存"""
  65. with self._lock:
  66. if port in self._port_to_egress:
  67. old_ip = self._port_to_egress.pop(port, None)
  68. VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
  69. def _async_probe_egress_ip(self, port: int, proto: str = "http"):
  70. """后台异步探测端口的真实公网出口 IP,不阻塞主调度"""
  71. with self._lock:
  72. if port in self._port_to_egress or port in self._probing_ports:
  73. return
  74. self._probing_ports.add(port)
  75. def _do_probe():
  76. try:
  77. proxy_url = f"{proto}://127.0.0.1:{port}"
  78. proxies = {"http": proxy_url, "https": proxy_url}
  79. # 尝试轻量公网 IP 端点
  80. resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
  81. if resp.status_code == 200 and resp.text.strip():
  82. ip = resp.text.strip()
  83. with self._lock:
  84. self._port_to_egress[port] = ip
  85. VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
  86. except Exception:
  87. pass
  88. finally:
  89. with self._lock:
  90. self._probing_ports.discard(port)
  91. t = threading.Thread(target=_do_probe, daemon=True)
  92. t.start()
  93. def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
  94. """获取节点的 IP 标识符(优先真实出口 IP,未解析则使用端口锚点)"""
  95. port = int(node.get("port", 0))
  96. configured_egress = node.get("egress_ip")
  97. if configured_egress:
  98. return configured_egress
  99. with self._lock:
  100. cached = self._port_to_egress.get(port)
  101. if cached:
  102. return cached
  103. if trigger_probe and port:
  104. proto = node.get("proto", "http")
  105. self._async_probe_egress_ip(port, proto)
  106. return f"port_{port}"
  107. # ==========================================
  108. # 云端信誉同步与防雪崩护栏
  109. # ==========================================
  110. def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
  111. """获取云端信誉字典 (绝不同步阻塞主流程,后台异步刷新)"""
  112. now = time.time()
  113. with self._lock:
  114. cache_entry = self._reputation_cache.get(plugin_name)
  115. if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
  116. return cache_entry.get("map", {})
  117. if plugin_name in self._fetching_reputation:
  118. return (cache_entry.get("map", {}) if cache_entry else {})
  119. self._fetching_reputation.add(plugin_name)
  120. def _fetch_worker():
  121. try:
  122. rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
  123. with self._lock:
  124. self._reputation_cache[plugin_name] = {
  125. "last_fetch": time.time(),
  126. "map": rep_map or {}
  127. }
  128. except Exception:
  129. pass
  130. finally:
  131. with self._lock:
  132. self._fetching_reputation.discard(plugin_name)
  133. threading.Thread(target=_fetch_worker, daemon=True).start()
  134. return (cache_entry.get("map", {}) if cache_entry else {})
  135. def _is_site_outage(self, plugin_name: str) -> bool:
  136. """
  137. Layer 1 防护:全站故障判决
  138. 过去 180s 内,跨至少 2 个不同 IP 且样本 >= 5,若失败率 >= 90% 或均报 502/503/520~526,判定为全站故障
  139. """
  140. with self._lock:
  141. history = self._sliding_history.get(plugin_name)
  142. if not history or len(history) < 5:
  143. return False
  144. now = time.time()
  145. recent_events = [e for e in history if now - e["ts"] < 180]
  146. if len(recent_events) < 5:
  147. return False
  148. distinct_ips = {e["egress_ip"] for e in recent_events}
  149. if len(distinct_ips) < 2:
  150. return False
  151. failures = [e for e in recent_events if not e["success"]]
  152. fail_rate = len(failures) / len(recent_events)
  153. # 严重系统级错误数量
  154. server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
  155. if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
  156. return True
  157. return False
  158. # ==========================================
  159. # 核心调度逻辑 (求交加权 + 五层护栏)
  160. # ==========================================
  161. def get_next_proxy(
  162. self,
  163. pools: List[str],
  164. proxy_cd: int = 60,
  165. plugin_name: Optional[str] = None
  166. ) -> Dict[str, Any]:
  167. """
  168. 根据 pools 列表解析并获取下一个代理。
  169. @param pools: 代理池列表,例如 ["local:local_multi_pool"] 或 ["cloud:proxy-cheap"]
  170. @param proxy_cd: 代理冷却时间(云端使用)
  171. @param plugin_name: 插件名称(如 'vfs_plugin'),用于结合信誉分和隔离策略
  172. @return dict 格式标准代理信息 (包含 id, proto, ip, port, username, password, egress_ip, name)
  173. """
  174. if getattr(configure, "TEST_PROXY", None):
  175. return configure.TEST_PROXY
  176. if not pools:
  177. raise BizLogicError(message="proxy_pool configuration list is empty.")
  178. local_targets = []
  179. cloud_targets = []
  180. for pool_item in pools:
  181. if not isinstance(pool_item, str):
  182. continue
  183. pool_item = pool_item.strip()
  184. if pool_item.startswith("local:"):
  185. local_pool_name = pool_item[len("local:"):].strip()
  186. if local_pool_name:
  187. local_targets.append(local_pool_name)
  188. elif pool_item.startswith("cloud:"):
  189. cloud_pool_name = pool_item[len("cloud:"):].strip()
  190. if cloud_pool_name:
  191. cloud_targets.append(cloud_pool_name)
  192. else:
  193. raise BizLogicError(
  194. message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
  195. )
  196. if local_targets:
  197. all_local_data = self._load_local_proxies()
  198. nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
  199. nodes_map = {
  200. n["name"]: n for n in nodes_list if isinstance(n, dict) and "name" in n
  201. }
  202. pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
  203. candidate_proxies: List[Dict[str, Any]] = []
  204. for lt in local_targets:
  205. pool_items = pools_dict.get(lt, [])
  206. if pool_items and isinstance(pool_items, list):
  207. for item in pool_items:
  208. if isinstance(item, str):
  209. node_obj = nodes_map.get(item)
  210. if node_obj:
  211. candidate_proxies.append(node_obj)
  212. elif isinstance(item, dict):
  213. candidate_proxies.append(item)
  214. if not candidate_proxies:
  215. raise BizLogicError(
  216. message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
  217. )
  218. # 记录节点名称缓存
  219. for node in candidate_proxies:
  220. port = int(node.get("port", 0))
  221. if port and "name" in node:
  222. self._port_to_name[port] = node["name"]
  223. # 如果没有指定 plugin_name,退化为经典随机轮询
  224. if not plugin_name:
  225. chosen = random.choice(candidate_proxies)
  226. return self._format_proxy_dict(chosen)
  227. # ------------------------------------------------------------------
  228. # 智能信誉分级调度 (求交 + 梯队路由 + 五层护栏)
  229. # ------------------------------------------------------------------
  230. now = time.time()
  231. rep_map = self._get_reputation_map(plugin_name)
  232. with self._lock:
  233. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  234. # 清理过期隔离节点
  235. expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
  236. for ip in expired_ips:
  237. quarantine_map.pop(ip, None)
  238. # Layer 3 防护:最低可用保底池检查 (保底 20% 或至少 5 个可用节点)
  239. total_nodes_count = len(candidate_proxies)
  240. quarantined_count = sum(
  241. 1 for node in candidate_proxies
  242. if self._resolve_ip_identifier(node) in quarantine_map
  243. )
  244. # 如果隔离比例超 70% 或剩余可用节点 < 20% (且小于 5),触发熔断重置
  245. if total_nodes_count > 0:
  246. available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
  247. if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
  248. VSC_WARN("vs_proxy", f"【代理防雪崩】插件 {plugin_name} 可用节点低于保底阈值(可用率 {available_ratio:.1%}),强制清空临时隔离名单!")
  249. quarantine_map.clear()
  250. # 划分三梯队
  251. trusted_tier: List[Dict[str, Any]] = []
  252. explore_tier: List[Dict[str, Any]] = []
  253. for node in candidate_proxies:
  254. ip_id = self._resolve_ip_identifier(node)
  255. # 检查是否在隔离名单中
  256. if ip_id in quarantine_map:
  257. continue
  258. # 检查云端信誉
  259. rep_info = rep_map.get(ip_id)
  260. if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
  261. trusted_tier.append(node)
  262. else:
  263. explore_tier.append(node)
  264. # 梯队加权调度:85% 优选,15% 探索
  265. chosen = None
  266. if trusted_tier and explore_tier:
  267. if random.random() < 0.85:
  268. chosen = random.choice(trusted_tier)
  269. else:
  270. chosen = random.choice(explore_tier)
  271. elif trusted_tier:
  272. chosen = random.choice(trusted_tier)
  273. elif explore_tier:
  274. chosen = random.choice(explore_tier)
  275. # Layer 5 防护:本地代码硬保底 (如果过滤后为空,回退全池随机)
  276. if not chosen:
  277. VSC_WARN("vs_proxy", f"【代理硬保底】插件 {plugin_name} 经过信誉过滤后无可用节点,触发全量随机兜底!")
  278. chosen = random.choice(candidate_proxies)
  279. return self._format_proxy_dict(chosen)
  280. if cloud_targets:
  281. return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
  282. raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
  283. def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
  284. """规范化代理字典"""
  285. port = int(chosen.get("port", 80))
  286. egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
  287. return {
  288. "id": chosen.get("id", 0),
  289. "proto": chosen.get("proto", "http"),
  290. "ip": chosen.get("ip", "127.0.0.1"),
  291. "port": port,
  292. "username": chosen.get("username", ""),
  293. "password": chosen.get("password", ""),
  294. "name": chosen.get("name", self._port_to_name.get(port, "")),
  295. "egress_ip": egress_ip,
  296. }
  297. # ==========================================
  298. # 指标埋点打桩与实时反馈 (Telemetry Upstream)
  299. # ==========================================
  300. def mark_proxy_result(
  301. self,
  302. proxy: Dict[str, Any],
  303. plugin_name: str,
  304. action: str,
  305. success: bool,
  306. http_code: int = 200,
  307. error_type: str = "",
  308. latency_ms: int = 0
  309. ):
  310. """
  311. 标记代理使用结果并异步上报。
  312. 内置:
  313. - 全站故障判决 (Site Outage)
  314. - 针对性阻断识别与临时隔离 (Quarantine)
  315. - 半开自愈复活 (Half-Open Revival)
  316. - 漂移检测 (Drift Detection)
  317. """
  318. if not proxy or not plugin_name:
  319. return
  320. port = int(proxy.get("port", 0))
  321. egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
  322. node_name = proxy.get("name") or self._port_to_name.get(port, "")
  323. # 1. 记入滑动调用历史
  324. now = time.time()
  325. with self._lock:
  326. history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
  327. history.append({
  328. "ts": now,
  329. "egress_ip": egress_ip,
  330. "success": success,
  331. "http_code": http_code,
  332. "error_type": error_type
  333. })
  334. # 2. 判断是否处于全站故障期
  335. is_outage = self._is_site_outage(plugin_name)
  336. with self._lock:
  337. quarantine_map = self._quarantine.setdefault(plugin_name, {})
  338. if success:
  339. # 成功事件:若在隔离期或半开期,立刻解除隔离并复活
  340. if egress_ip in quarantine_map:
  341. quarantine_map.pop(egress_ip, None)
  342. VSC_INFO("vs_proxy", f"【代理自愈复活】出口 IP {egress_ip} (Port {port}) 在插件 {plugin_name} 验证成功,已移出隔离名单!")
  343. else:
  344. # 失败事件
  345. if is_outage:
  346. VSC_WARN("vs_proxy", f"【全站故障保护】目标网站疑似出现全站级故障,已冻结对出口 IP {egress_ip} 的扣分与隔离!")
  347. else:
  348. # Layer 2: 错误精细分类判定
  349. is_waf_block = (
  350. "cf-mitigated" in error_type.lower() or
  351. "403202" in error_type or
  352. "waf blocked" in error_type.lower() or
  353. (http_code == 403 and "mitigated" in error_type.lower())
  354. )
  355. if is_waf_block:
  356. # 针对性阻断:进入隔离名单(默认 1800 秒)
  357. quarantine_map[egress_ip] = now + 1800
  358. VSC_WARN("vs_proxy", f"【节点临时隔离】出口 IP {egress_ip} (Port {port}) 触发针对性 WAF 阻断,临时隔离 30 分钟!")
  359. # 触发潜在漂移检测:清除缓存以便重新探查真实公网 IP 是否已变
  360. self.invalidate_egress_cache(port)
  361. # 3. 异步上报埋点指标至云端 (Fail-Silent)
  362. metric_payload = {
  363. "client_id": getattr(configure, "NODE_ID", "local_node"),
  364. "plugin_name": plugin_name,
  365. "local_port": port,
  366. "node_name": node_name,
  367. "egress_ip": egress_ip,
  368. "action": action,
  369. "success": success,
  370. "http_code": http_code,
  371. "error_type": error_type,
  372. "latency_ms": latency_ms,
  373. "timestamp": int(now)
  374. }
  375. def _report_worker():
  376. try:
  377. CloudApi.Instance().report_proxy_metric(metric_payload)
  378. except Exception:
  379. pass
  380. t = threading.Thread(target=_report_worker, daemon=True)
  381. t.start()