|
|
@@ -1,22 +1,29 @@
|
|
|
import os
|
|
|
+import time
|
|
|
import json
|
|
|
import yaml
|
|
|
import random
|
|
|
+import threading
|
|
|
+import collections
|
|
|
+import concurrent.futures
|
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
|
|
+import requests
|
|
|
import configure
|
|
|
from toolkit.cloud_api import CloudApi
|
|
|
from core_types import BizLogicError
|
|
|
-from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR
|
|
|
+from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
|
|
|
|
|
|
|
|
|
class ProxyManager:
|
|
|
"""
|
|
|
- 统一代理调度管理器。
|
|
|
- 支持显式前缀区分:
|
|
|
- - "local:<pool_name>" -> 从本地 config/proxies.yaml 随机分配
|
|
|
- - "cloud:<pool_name>" -> 通过 CloudApi 请求云端代理池
|
|
|
- 严格校验:未带合法前缀将抛出异常。
|
|
|
+ 统一代理调度与信誉管理中心。
|
|
|
+ 支持:
|
|
|
+ - 显式前缀区分:"local:<pool_name>" / "cloud:<pool_name>"
|
|
|
+ - 出口 IP 探针与缓存机制 (Egress Resolver & Cache)
|
|
|
+ - 本地通道与云端信誉库求交加权调度 (85% 优选 + 15% 探索)
|
|
|
+ - 五层防雪崩护栏 (全站故障冻结、20% 最低保底池、半开自愈、底层硬兜底)
|
|
|
+ - 异步指标埋点与上报 (Fail-Silent)
|
|
|
"""
|
|
|
_instance = None
|
|
|
|
|
|
@@ -34,6 +41,23 @@ class ProxyManager:
|
|
|
self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
|
|
|
|
|
|
+ self._lock = threading.RLock()
|
|
|
+
|
|
|
+ # 端口 -> 公网出口 IP 缓存与节点名称缓存
|
|
|
+ self._port_to_egress: Dict[int, str] = {}
|
|
|
+ self._port_to_name: Dict[int, str] = {}
|
|
|
+ self._probing_ports = set()
|
|
|
+
|
|
|
+ # 插件维度本地临时隔离黑名单: { plugin_name: { egress_ip: expire_ts } }
|
|
|
+ self._quarantine: Dict[str, Dict[str, float]] = {}
|
|
|
+
|
|
|
+ # 插件维度云端信誉缓存: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
|
|
|
+ self._reputation_cache: Dict[str, Dict[str, Any]] = {}
|
|
|
+ self._fetching_reputation = set()
|
|
|
+
|
|
|
+ # 插件维度滑动调用历史 (用于全站故障判定): { plugin_name: collections.deque(maxlen=30) }
|
|
|
+ self._sliding_history: Dict[str, collections.deque] = {}
|
|
|
+
|
|
|
def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
|
|
|
if not os.path.exists(self.local_proxies_file):
|
|
|
raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
|
|
|
@@ -43,14 +67,147 @@ class ProxyManager:
|
|
|
except Exception as e:
|
|
|
raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
|
|
|
|
|
|
- def get_next_proxy(self, pools: List[str], proxy_cd: int = 60) -> Dict[str, Any]:
|
|
|
+ # ==========================================
|
|
|
+ # 出口 IP 探针与缓存 (Egress Resolver)
|
|
|
+ # ==========================================
|
|
|
+
|
|
|
+ def get_cached_egress_ip(self, port: int) -> Optional[str]:
|
|
|
+ with self._lock:
|
|
|
+ return self._port_to_egress.get(port)
|
|
|
+
|
|
|
+ def invalidate_egress_cache(self, port: int):
|
|
|
+ """当节点疑似漂移时清除缓存"""
|
|
|
+ with self._lock:
|
|
|
+ if port in self._port_to_egress:
|
|
|
+ old_ip = self._port_to_egress.pop(port, None)
|
|
|
+ VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
|
|
|
+
|
|
|
+ def _async_probe_egress_ip(self, port: int, proto: str = "http"):
|
|
|
+ """后台异步探测端口的真实公网出口 IP,不阻塞主调度"""
|
|
|
+ with self._lock:
|
|
|
+ if port in self._port_to_egress or port in self._probing_ports:
|
|
|
+ return
|
|
|
+ self._probing_ports.add(port)
|
|
|
+
|
|
|
+ def _do_probe():
|
|
|
+ try:
|
|
|
+ proxy_url = f"{proto}://127.0.0.1:{port}"
|
|
|
+ proxies = {"http": proxy_url, "https": proxy_url}
|
|
|
+ # 尝试轻量公网 IP 端点
|
|
|
+ resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
|
|
|
+ if resp.status_code == 200 and resp.text.strip():
|
|
|
+ ip = resp.text.strip()
|
|
|
+ with self._lock:
|
|
|
+ self._port_to_egress[port] = ip
|
|
|
+ VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+ finally:
|
|
|
+ with self._lock:
|
|
|
+ self._probing_ports.discard(port)
|
|
|
+
|
|
|
+ t = threading.Thread(target=_do_probe, daemon=True)
|
|
|
+ t.start()
|
|
|
+
|
|
|
+ def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
|
|
|
+ """获取节点的 IP 标识符(优先真实出口 IP,未解析则使用端口锚点)"""
|
|
|
+ port = int(node.get("port", 0))
|
|
|
+ configured_egress = node.get("egress_ip")
|
|
|
+ if configured_egress:
|
|
|
+ return configured_egress
|
|
|
+
|
|
|
+ with self._lock:
|
|
|
+ cached = self._port_to_egress.get(port)
|
|
|
+ if cached:
|
|
|
+ return cached
|
|
|
+
|
|
|
+ if trigger_probe and port:
|
|
|
+ proto = node.get("proto", "http")
|
|
|
+ self._async_probe_egress_ip(port, proto)
|
|
|
+
|
|
|
+ return f"port_{port}"
|
|
|
+
|
|
|
+ # ==========================================
|
|
|
+ # 云端信誉同步与防雪崩护栏
|
|
|
+ # ==========================================
|
|
|
+
|
|
|
+ def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
|
|
|
+ """获取云端信誉字典 (绝不同步阻塞主流程,后台异步刷新)"""
|
|
|
+ now = time.time()
|
|
|
+ with self._lock:
|
|
|
+ cache_entry = self._reputation_cache.get(plugin_name)
|
|
|
+ if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
|
|
|
+ return cache_entry.get("map", {})
|
|
|
+
|
|
|
+ if plugin_name in self._fetching_reputation:
|
|
|
+ return (cache_entry.get("map", {}) if cache_entry else {})
|
|
|
+
|
|
|
+ self._fetching_reputation.add(plugin_name)
|
|
|
+
|
|
|
+ def _fetch_worker():
|
|
|
+ try:
|
|
|
+ rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
|
|
|
+ with self._lock:
|
|
|
+ self._reputation_cache[plugin_name] = {
|
|
|
+ "last_fetch": time.time(),
|
|
|
+ "map": rep_map or {}
|
|
|
+ }
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+ finally:
|
|
|
+ with self._lock:
|
|
|
+ self._fetching_reputation.discard(plugin_name)
|
|
|
+
|
|
|
+ threading.Thread(target=_fetch_worker, daemon=True).start()
|
|
|
+ return (cache_entry.get("map", {}) if cache_entry else {})
|
|
|
+
|
|
|
+ def _is_site_outage(self, plugin_name: str) -> bool:
|
|
|
+ """
|
|
|
+ Layer 1 防护:全站故障判决
|
|
|
+ 过去 180s 内,跨至少 2 个不同 IP 且样本 >= 5,若失败率 >= 90% 或均报 502/503/520~526,判定为全站故障
|
|
|
+ """
|
|
|
+ with self._lock:
|
|
|
+ history = self._sliding_history.get(plugin_name)
|
|
|
+ if not history or len(history) < 5:
|
|
|
+ return False
|
|
|
+
|
|
|
+ now = time.time()
|
|
|
+ recent_events = [e for e in history if now - e["ts"] < 180]
|
|
|
+ if len(recent_events) < 5:
|
|
|
+ return False
|
|
|
+
|
|
|
+ distinct_ips = {e["egress_ip"] for e in recent_events}
|
|
|
+ if len(distinct_ips) < 2:
|
|
|
+ return False
|
|
|
+
|
|
|
+ failures = [e for e in recent_events if not e["success"]]
|
|
|
+ fail_rate = len(failures) / len(recent_events)
|
|
|
+
|
|
|
+ # 严重系统级错误数量
|
|
|
+ server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
|
|
|
+
|
|
|
+ if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
|
|
|
+ return True
|
|
|
+
|
|
|
+ return False
|
|
|
+
|
|
|
+ # ==========================================
|
|
|
+ # 核心调度逻辑 (求交加权 + 五层护栏)
|
|
|
+ # ==========================================
|
|
|
+
|
|
|
+ def get_next_proxy(
|
|
|
+ self,
|
|
|
+ pools: List[str],
|
|
|
+ proxy_cd: int = 60,
|
|
|
+ plugin_name: Optional[str] = None
|
|
|
+ ) -> Dict[str, Any]:
|
|
|
"""
|
|
|
根据 pools 列表解析并获取下一个代理。
|
|
|
@param pools: 代理池列表,例如 ["local:local_multi_pool"] 或 ["cloud:proxy-cheap"]
|
|
|
@param proxy_cd: 代理冷却时间(云端使用)
|
|
|
- @return dict 格式标准代理信息 (包含 id, proto, ip, port, username, password)
|
|
|
+ @param plugin_name: 插件名称(如 'vfs_plugin'),用于结合信誉分和隔离策略
|
|
|
+ @return dict 格式标准代理信息 (包含 id, proto, ip, port, username, password, egress_ip, name)
|
|
|
"""
|
|
|
- # 如果配置了全局静态测试代理则优先使用
|
|
|
if getattr(configure, "TEST_PROXY", None):
|
|
|
return configure.TEST_PROXY
|
|
|
|
|
|
@@ -77,7 +234,6 @@ class ProxyManager:
|
|
|
message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
|
|
|
)
|
|
|
|
|
|
- # 优先支持本地代理池
|
|
|
if local_targets:
|
|
|
all_local_data = self._load_local_proxies()
|
|
|
nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
|
|
|
@@ -87,7 +243,7 @@ class ProxyManager:
|
|
|
pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
|
|
|
|
|
|
candidate_proxies: List[Dict[str, Any]] = []
|
|
|
-
|
|
|
+
|
|
|
for lt in local_targets:
|
|
|
pool_items = pools_dict.get(lt, [])
|
|
|
if pool_items and isinstance(pool_items, list):
|
|
|
@@ -104,21 +260,191 @@ class ProxyManager:
|
|
|
message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
|
|
|
)
|
|
|
|
|
|
- chosen = random.choice(candidate_proxies)
|
|
|
-
|
|
|
- # 规范化代理字典,确保与 Proxy 字段完全契合
|
|
|
- proxy_dict = {
|
|
|
- "id": chosen.get("id", 0),
|
|
|
- "proto": chosen.get("proto", "http"),
|
|
|
- "ip": chosen.get("ip", "127.0.0.1"),
|
|
|
- "port": int(chosen.get("port", 80)),
|
|
|
- "username": chosen.get("username", ""),
|
|
|
- "password": chosen.get("password", "")
|
|
|
- }
|
|
|
- return proxy_dict
|
|
|
+ # 记录节点名称缓存
|
|
|
+ for node in candidate_proxies:
|
|
|
+ port = int(node.get("port", 0))
|
|
|
+ if port and "name" in node:
|
|
|
+ self._port_to_name[port] = node["name"]
|
|
|
+
|
|
|
+ # 如果没有指定 plugin_name,退化为经典随机轮询
|
|
|
+ if not plugin_name:
|
|
|
+ chosen = random.choice(candidate_proxies)
|
|
|
+ return self._format_proxy_dict(chosen)
|
|
|
+
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+ # 智能信誉分级调度 (求交 + 梯队路由 + 五层护栏)
|
|
|
+ # ------------------------------------------------------------------
|
|
|
+ now = time.time()
|
|
|
+ rep_map = self._get_reputation_map(plugin_name)
|
|
|
+
|
|
|
+ with self._lock:
|
|
|
+ quarantine_map = self._quarantine.setdefault(plugin_name, {})
|
|
|
+ # 清理过期隔离节点
|
|
|
+ expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
|
|
|
+ for ip in expired_ips:
|
|
|
+ quarantine_map.pop(ip, None)
|
|
|
+
|
|
|
+ # Layer 3 防护:最低可用保底池检查 (保底 20% 或至少 5 个可用节点)
|
|
|
+ total_nodes_count = len(candidate_proxies)
|
|
|
+ quarantined_count = sum(
|
|
|
+ 1 for node in candidate_proxies
|
|
|
+ if self._resolve_ip_identifier(node) in quarantine_map
|
|
|
+ )
|
|
|
+
|
|
|
+ # 如果隔离比例超 70% 或剩余可用节点 < 20% (且小于 5),触发熔断重置
|
|
|
+ if total_nodes_count > 0:
|
|
|
+ available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
|
|
|
+ if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
|
|
|
+ VSC_WARN("vs_proxy", f"【代理防雪崩】插件 {plugin_name} 可用节点低于保底阈值(可用率 {available_ratio:.1%}),强制清空临时隔离名单!")
|
|
|
+ quarantine_map.clear()
|
|
|
+
|
|
|
+ # 划分三梯队
|
|
|
+ trusted_tier: List[Dict[str, Any]] = []
|
|
|
+ explore_tier: List[Dict[str, Any]] = []
|
|
|
+
|
|
|
+ for node in candidate_proxies:
|
|
|
+ ip_id = self._resolve_ip_identifier(node)
|
|
|
+ # 检查是否在隔离名单中
|
|
|
+ if ip_id in quarantine_map:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 检查云端信誉
|
|
|
+ rep_info = rep_map.get(ip_id)
|
|
|
+ if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
|
|
|
+ trusted_tier.append(node)
|
|
|
+ else:
|
|
|
+ explore_tier.append(node)
|
|
|
+
|
|
|
+ # 梯队加权调度:85% 优选,15% 探索
|
|
|
+ chosen = None
|
|
|
+ if trusted_tier and explore_tier:
|
|
|
+ if random.random() < 0.85:
|
|
|
+ chosen = random.choice(trusted_tier)
|
|
|
+ else:
|
|
|
+ chosen = random.choice(explore_tier)
|
|
|
+ elif trusted_tier:
|
|
|
+ chosen = random.choice(trusted_tier)
|
|
|
+ elif explore_tier:
|
|
|
+ chosen = random.choice(explore_tier)
|
|
|
+
|
|
|
+ # Layer 5 防护:本地代码硬保底 (如果过滤后为空,回退全池随机)
|
|
|
+ if not chosen:
|
|
|
+ VSC_WARN("vs_proxy", f"【代理硬保底】插件 {plugin_name} 经过信誉过滤后无可用节点,触发全量随机兜底!")
|
|
|
+ chosen = random.choice(candidate_proxies)
|
|
|
+
|
|
|
+ return self._format_proxy_dict(chosen)
|
|
|
|
|
|
- # 如果指定的是云端代理池
|
|
|
if cloud_targets:
|
|
|
return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
|
|
|
|
|
|
raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
|
|
|
+
|
|
|
+ def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
+ """规范化代理字典"""
|
|
|
+ port = int(chosen.get("port", 80))
|
|
|
+ egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
|
|
|
+ return {
|
|
|
+ "id": chosen.get("id", 0),
|
|
|
+ "proto": chosen.get("proto", "http"),
|
|
|
+ "ip": chosen.get("ip", "127.0.0.1"),
|
|
|
+ "port": port,
|
|
|
+ "username": chosen.get("username", ""),
|
|
|
+ "password": chosen.get("password", ""),
|
|
|
+ "name": chosen.get("name", self._port_to_name.get(port, "")),
|
|
|
+ "egress_ip": egress_ip,
|
|
|
+ }
|
|
|
+
|
|
|
+ # ==========================================
|
|
|
+ # 指标埋点打桩与实时反馈 (Telemetry Upstream)
|
|
|
+ # ==========================================
|
|
|
+
|
|
|
+ def mark_proxy_result(
|
|
|
+ self,
|
|
|
+ proxy: Dict[str, Any],
|
|
|
+ plugin_name: str,
|
|
|
+ action: str,
|
|
|
+ success: bool,
|
|
|
+ http_code: int = 200,
|
|
|
+ error_type: str = "",
|
|
|
+ latency_ms: int = 0
|
|
|
+ ):
|
|
|
+ """
|
|
|
+ 标记代理使用结果并异步上报。
|
|
|
+ 内置:
|
|
|
+ - 全站故障判决 (Site Outage)
|
|
|
+ - 针对性阻断识别与临时隔离 (Quarantine)
|
|
|
+ - 半开自愈复活 (Half-Open Revival)
|
|
|
+ - 漂移检测 (Drift Detection)
|
|
|
+ """
|
|
|
+ if not proxy or not plugin_name:
|
|
|
+ return
|
|
|
+
|
|
|
+ port = int(proxy.get("port", 0))
|
|
|
+ egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
|
|
|
+ node_name = proxy.get("name") or self._port_to_name.get(port, "")
|
|
|
+
|
|
|
+ # 1. 记入滑动调用历史
|
|
|
+ now = time.time()
|
|
|
+ with self._lock:
|
|
|
+ history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
|
|
|
+ history.append({
|
|
|
+ "ts": now,
|
|
|
+ "egress_ip": egress_ip,
|
|
|
+ "success": success,
|
|
|
+ "http_code": http_code,
|
|
|
+ "error_type": error_type
|
|
|
+ })
|
|
|
+
|
|
|
+ # 2. 判断是否处于全站故障期
|
|
|
+ is_outage = self._is_site_outage(plugin_name)
|
|
|
+
|
|
|
+ with self._lock:
|
|
|
+ quarantine_map = self._quarantine.setdefault(plugin_name, {})
|
|
|
+
|
|
|
+ if success:
|
|
|
+ # 成功事件:若在隔离期或半开期,立刻解除隔离并复活
|
|
|
+ if egress_ip in quarantine_map:
|
|
|
+ quarantine_map.pop(egress_ip, None)
|
|
|
+ VSC_INFO("vs_proxy", f"【代理自愈复活】出口 IP {egress_ip} (Port {port}) 在插件 {plugin_name} 验证成功,已移出隔离名单!")
|
|
|
+ else:
|
|
|
+ # 失败事件
|
|
|
+ if is_outage:
|
|
|
+ VSC_WARN("vs_proxy", f"【全站故障保护】目标网站疑似出现全站级故障,已冻结对出口 IP {egress_ip} 的扣分与隔离!")
|
|
|
+ else:
|
|
|
+ # Layer 2: 错误精细分类判定
|
|
|
+ is_waf_block = (
|
|
|
+ "cf-mitigated" in error_type.lower() or
|
|
|
+ "403202" in error_type or
|
|
|
+ "waf blocked" in error_type.lower() or
|
|
|
+ (http_code == 403 and "mitigated" in error_type.lower())
|
|
|
+ )
|
|
|
+ if is_waf_block:
|
|
|
+ # 针对性阻断:进入隔离名单(默认 1800 秒)
|
|
|
+ quarantine_map[egress_ip] = now + 1800
|
|
|
+ VSC_WARN("vs_proxy", f"【节点临时隔离】出口 IP {egress_ip} (Port {port}) 触发针对性 WAF 阻断,临时隔离 30 分钟!")
|
|
|
+ # 触发潜在漂移检测:清除缓存以便重新探查真实公网 IP 是否已变
|
|
|
+ self.invalidate_egress_cache(port)
|
|
|
+
|
|
|
+ # 3. 异步上报埋点指标至云端 (Fail-Silent)
|
|
|
+ metric_payload = {
|
|
|
+ "client_id": getattr(configure, "NODE_ID", "local_node"),
|
|
|
+ "plugin_name": plugin_name,
|
|
|
+ "local_port": port,
|
|
|
+ "node_name": node_name,
|
|
|
+ "egress_ip": egress_ip,
|
|
|
+ "action": action,
|
|
|
+ "success": success,
|
|
|
+ "http_code": http_code,
|
|
|
+ "error_type": error_type,
|
|
|
+ "latency_ms": latency_ms,
|
|
|
+ "timestamp": int(now)
|
|
|
+ }
|
|
|
+
|
|
|
+ def _report_worker():
|
|
|
+ try:
|
|
|
+ CloudApi.Instance().report_proxy_metric(metric_payload)
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+
|
|
|
+ t = threading.Thread(target=_report_worker, daemon=True)
|
|
|
+ t.start()
|