|
|
@@ -6,7 +6,7 @@ import random
|
|
|
import threading
|
|
|
import collections
|
|
|
import concurrent.futures
|
|
|
-from typing import List, Dict, Any, Optional
|
|
|
+from typing import List, Dict, Any, Optional, Tuple
|
|
|
|
|
|
import requests
|
|
|
import configure
|
|
|
@@ -59,6 +59,9 @@ class ProxyManager:
|
|
|
# Per-plugin sliding call history (for site-wide outage determination): { plugin_name: collections.deque(maxlen=30) }
|
|
|
self._sliding_history: Dict[str, collections.deque] = {}
|
|
|
|
|
|
+ # Per-plugin node cooldown / last-used tracking: { plugin_name: { identifier: expire_ts } }
|
|
|
+ self._node_cooldown: Dict[str, Dict[str, float]] = {}
|
|
|
+
|
|
|
def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
|
|
|
if not os.path.exists(self.local_proxies_file):
|
|
|
raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
|
|
|
@@ -281,19 +284,27 @@ class ProxyManager:
|
|
|
# ------------------------------------------------------------------
|
|
|
now = time.time()
|
|
|
rep_map = self._get_reputation_map(plugin_name)
|
|
|
+ effective_cd = max(float(proxy_cd or 60.0), 30.0)
|
|
|
|
|
|
with self._lock:
|
|
|
quarantine_map = self._quarantine.setdefault(plugin_name, {})
|
|
|
+ cooldown_map = self._node_cooldown.setdefault(plugin_name, {})
|
|
|
+
|
|
|
# Clean up expired quarantine entries
|
|
|
expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
|
|
|
for ip in expired_ips:
|
|
|
quarantine_map.pop(ip, None)
|
|
|
|
|
|
+ # Clean up expired cooldown entries
|
|
|
+ expired_cd = [k for k, exp in cooldown_map.items() if now > exp]
|
|
|
+ for k in expired_cd:
|
|
|
+ cooldown_map.pop(k, None)
|
|
|
+
|
|
|
# Layer 3 protection: Minimum available pool guarantee (20% or at least 5 nodes)
|
|
|
total_nodes_count = len(candidate_proxies)
|
|
|
quarantined_count = sum(
|
|
|
1 for node in candidate_proxies
|
|
|
- if self._resolve_ip_identifier(node) in quarantine_map
|
|
|
+ if self._resolve_ip_identifier(node) in quarantine_map or f"port_{int(node.get('port', 0))}" in quarantine_map
|
|
|
)
|
|
|
|
|
|
# If quarantine ratio > 70% or available nodes < 20% (and < 5), trigger circuit-breaker reset
|
|
|
@@ -308,35 +319,62 @@ class ProxyManager:
|
|
|
explore_tier: List[Dict[str, Any]] = []
|
|
|
|
|
|
for node in candidate_proxies:
|
|
|
+ port = int(node.get("port", 0))
|
|
|
ip_id = self._resolve_ip_identifier(node)
|
|
|
- # Check if in quarantine
|
|
|
- if ip_id in quarantine_map:
|
|
|
+ # Check if in quarantine (check both ip_id and port)
|
|
|
+ if ip_id in quarantine_map or (port and f"port_{port}" in quarantine_map):
|
|
|
continue
|
|
|
|
|
|
- # Check cloud reputation
|
|
|
+ # Check cloud reputation (check both ip_id and port)
|
|
|
rep_info = rep_map.get(ip_id)
|
|
|
+ if not rep_info and port:
|
|
|
+ rep_info = rep_map.get(f"port_{port}")
|
|
|
if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
|
|
|
trusted_tier.append(node)
|
|
|
else:
|
|
|
explore_tier.append(node)
|
|
|
|
|
|
- # Tier weighted scheduling: 85% preferred, 15% exploration
|
|
|
- chosen = None
|
|
|
+ # Proportional dynamic trust factor:
|
|
|
+ # Trusted tier ratio dynamically scales with the number of trusted nodes (up to max 0.85).
|
|
|
+ # When only 1 node is trusted, it gets at most 20% of traffic, ensuring 80% explores other nodes to avoid single-node monopolization.
|
|
|
+ trusted_ratio = min(0.85, 0.20 * len(trusted_tier)) if (trusted_tier and explore_tier) else (1.0 if trusted_tier else 0.0)
|
|
|
+
|
|
|
+ target_tier: List[Dict[str, Any]] = []
|
|
|
if trusted_tier and explore_tier:
|
|
|
- if random.random() < 0.85:
|
|
|
- chosen = random.choice(trusted_tier)
|
|
|
- else:
|
|
|
- chosen = random.choice(explore_tier)
|
|
|
+ target_tier = trusted_tier if (random.random() < trusted_ratio) else explore_tier
|
|
|
elif trusted_tier:
|
|
|
- chosen = random.choice(trusted_tier)
|
|
|
+ target_tier = trusted_tier
|
|
|
elif explore_tier:
|
|
|
- chosen = random.choice(explore_tier)
|
|
|
-
|
|
|
- # Layer 5 protection: Hard fallback (if empty after filtering, fallback to full pool random)
|
|
|
- if not chosen:
|
|
|
- VSC_WARN("vs_proxy", f"[Proxy Hard Fallback] Plugin {plugin_name} has no available nodes after reputation filter. Triggering full random fallback!")
|
|
|
+ target_tier = explore_tier
|
|
|
+ else:
|
|
|
+ target_tier = candidate_proxies
|
|
|
+
|
|
|
+ # Anti-Affinity & Local Cooldown Selection:
|
|
|
+ # Prefer non-cooling nodes. If all nodes in tier are cooling, choose the least-recently used.
|
|
|
+ def _is_cooling(node: Dict[str, Any]) -> Tuple[bool, float]:
|
|
|
+ p = int(node.get("port", 0))
|
|
|
+ iid = self._resolve_ip_identifier(node)
|
|
|
+ cd_exp = max(cooldown_map.get(iid, 0), cooldown_map.get(f"port_{p}", 0) if p else 0)
|
|
|
+ return (cd_exp > now, cd_exp)
|
|
|
+
|
|
|
+ non_cooling = [n for n in target_tier if not _is_cooling(n)[0]]
|
|
|
+ if non_cooling:
|
|
|
+ chosen = random.choice(non_cooling)
|
|
|
+ elif target_tier:
|
|
|
+ # All candidates in tier are cooling, pick the one that expires earliest
|
|
|
+ chosen = min(target_tier, key=lambda n: _is_cooling(n)[1])
|
|
|
+ else:
|
|
|
chosen = random.choice(candidate_proxies)
|
|
|
|
|
|
+ # Record cooldown on the chosen node
|
|
|
+ chosen_port = int(chosen.get("port", 0))
|
|
|
+ chosen_id = self._resolve_ip_identifier(chosen)
|
|
|
+ cooldown_until = now + effective_cd
|
|
|
+ with self._lock:
|
|
|
+ cooldown_map[chosen_id] = cooldown_until
|
|
|
+ if chosen_port:
|
|
|
+ cooldown_map[f"port_{chosen_port}"] = cooldown_until
|
|
|
+
|
|
|
return self._format_proxy_dict(chosen)
|
|
|
|
|
|
if cloud_targets:
|
|
|
@@ -406,14 +444,26 @@ class ProxyManager:
|
|
|
|
|
|
with self._lock:
|
|
|
quarantine_map = self._quarantine.setdefault(plugin_name, {})
|
|
|
+ cooldown_map = self._node_cooldown.setdefault(plugin_name, {})
|
|
|
|
|
|
if success:
|
|
|
# Success event: If quarantined or half-open, immediately revive and unquarantine
|
|
|
+ revived = False
|
|
|
if egress_ip in quarantine_map:
|
|
|
quarantine_map.pop(egress_ip, None)
|
|
|
+ revived = True
|
|
|
+ if port and f"port_{port}" in quarantine_map:
|
|
|
+ quarantine_map.pop(f"port_{port}", None)
|
|
|
+ revived = True
|
|
|
+ if revived:
|
|
|
VSC_INFO("vs_proxy", f"[Proxy Half-Open Revival] Egress IP {egress_ip} (Port {port}) succeeded for plugin {plugin_name}. Removed from quarantine!")
|
|
|
else:
|
|
|
- # Failure event
|
|
|
+ # Failure event: Always apply transient failure cooldown (180s) to prevent immediate re-hammering
|
|
|
+ fail_cd_until = now + 180.0
|
|
|
+ cooldown_map[egress_ip] = max(cooldown_map.get(egress_ip, 0), fail_cd_until)
|
|
|
+ if port:
|
|
|
+ cooldown_map[f"port_{port}"] = max(cooldown_map.get(f"port_{port}", 0), fail_cd_until)
|
|
|
+
|
|
|
if is_outage:
|
|
|
VSC_WARN("vs_proxy", f"[Site Outage Protection] Target site suspected of site-wide outage. Freezing penalties and quarantine for egress IP {egress_ip}!")
|
|
|
else:
|
|
|
@@ -423,12 +473,17 @@ class ProxyManager:
|
|
|
"403201" in error_type or
|
|
|
"403202" in error_type or
|
|
|
"waf blocked" in error_type.lower() or
|
|
|
+ "turnstile" in error_type.lower() or
|
|
|
+ "challenge" in error_type.lower() or
|
|
|
+ "cloudflare" in error_type.lower() or
|
|
|
(http_code == 403 and "mitigated" in error_type.lower())
|
|
|
)
|
|
|
if is_waf_block:
|
|
|
# Targeted block: enter quarantine list (default 1800s)
|
|
|
quarantine_map[egress_ip] = now + 1800
|
|
|
- VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF block. Quarantined for 30 minutes!")
|
|
|
+ if port:
|
|
|
+ quarantine_map[f"port_{port}"] = now + 1800
|
|
|
+ VSC_WARN("vs_proxy", f"[Node Quarantined] Egress IP {egress_ip} (Port {port}) triggered targeted WAF/Turnstile block. Quarantined for 30 minutes!")
|
|
|
# Trigger potential drift detection: invalidate cache to re-probe true public IP
|
|
|
self.invalidate_egress_cache(port)
|
|
|
|