Przeglądaj źródła

feat: implement proxy reputation scheduling, telemetry metrics, and safe account release

hujiarui 1 miesiąc temu
rodzic
commit
95ddf8eb4b

+ 68 - 14
booker.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from typing import List, Dict, Callable, Optional
 
 from core_types import (
-    GroupConfig, PluginConfig, Task, BookResult, QueryResult, 
+    GroupConfig, PluginConfig, Account, Task, BookResult, QueryResult, 
     AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
     BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
     NetworkTimeoutError
@@ -152,9 +152,12 @@ class BaseBookerGCO:
 
     def _normalize_error(self, err: Exception) -> BizException:
         """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
+        err_str = str(err)
+        # 如果是底层网络/CORS预检被拒导致的 Fetch 失败或 WAF 阻断,精准提升为 PermissionDeniedError
+        if "Failed to fetch" in err_str or "Status: 0" in err_str or "WAF Blocked" in err_str or "cf-mitigated" in err_str:
+            return PermissionDeniedError(message=err_str)
         if isinstance(err, BizException):
             return err
-        err_str = str(err)
         if "42901" in err_str or "Rate limited" in err_str:
             return RateLimiteddError(message=err_str)
         if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
@@ -166,6 +169,15 @@ class BaseBookerGCO:
             message=f"Unhandled ({type(err).__name__}): {err_str}",
         )
 
+    def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
+        """安全归还账号:将云端冷却时间重置为 0,避免非账号问题导致账号被误锁 3 小时"""
+        if acc and acc.id > 0 and acc.username not in ("", "Guest"):
+            try:
+                CloudApi.Instance().lock_account(acc.id, 0)
+                self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
+            except Exception as e:
+                self._log(f"Failed to release account [{acc.username}]: {e}")
+
     def _dispatch_error_policy(
         self,
         action: str,
@@ -180,22 +192,19 @@ class BaseBookerGCO:
         - spawn/login 时 429:判定为账号级限流,锁定账号,更新 Redis 订单/任务追踪器与生成冷却;
         - book 时 429:档期提交频控或出口 IP 限制,不锁定账号,移除任务以轮换代理或重试;
         - session expired / 401:会话失效,移除任务;
-        - 403 Forbidden / WAF:判定为 IP 受阻,移除任务以轮换代理;
+        - 403 Forbidden / WAF:判定为 IP 受阻,移除任务以轮换代理;在 spawn 时立即安全释放账号;
         - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
         """
         biz_err = self._normalize_error(err)
-        acc = None
-        if plg_cfg and plg_cfg.account:
-            acc = plg_cfg.account
-        elif task and getattr(task.instance, "config", None) and task.instance.config.account:
-            acc = task.instance.config.account
+        active_plg_cfg = plg_cfg or (task.instance.config if task else None)
+        acc = active_plg_cfg.account if active_plg_cfg else None
 
         # 1. 目标站限流 / 429
         if isinstance(biz_err, RateLimiteddError):
             if action in ("spawn", "login", "create_session"):
                 backoff = float(self.m_cfg.login_backoff)
                 self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
-                if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0):
+                if acc and acc.username not in ("", "Guest") and acc.id > 0:
                     try:
                         CloudApi.Instance().lock_account(acc.id, int(backoff))
                         self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
@@ -232,11 +241,12 @@ class BaseBookerGCO:
 
         # 3. 权限被拒 / WAF 拦截 / 403
         elif isinstance(biz_err, PermissionDeniedError):
-            proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)
+            proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
             self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
             if task:
                 self._remove_task(task, f"{action} waf blocked")
-            if action == "spawn":
+            if action in ("spawn", "login", "create_session"):
+                self._release_account_safely(acc, reason="WAF Blocked")
                 self.m_spawn_cooldown_until = time.time() + proxy_cd
 
         # 4. 网络或浏览器超时
@@ -244,15 +254,35 @@ class BaseBookerGCO:
             self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
             if task:
                 task.next_run = time.time() + 5.0
+            if action in ("spawn", "login", "create_session"):
+                self._release_account_safely(acc, reason="Network Timeout")
+                self.m_spawn_cooldown_until = time.time() + 10.0
 
         # 5. 兜底策略
         else:
             self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
-            if action == "spawn":
+            if action in ("spawn", "login", "create_session"):
+                self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
                 self.m_spawn_cooldown_until = time.time() + 10.0
             elif task:
                 task.next_run = time.time() + 10.0
 
+        # 6. 统一代理健康指标埋点
+        if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
+            err_type = biz_err.message
+            if isinstance(biz_err, PermissionDeniedError):
+                if "403202" in err_type or "cf-mitigated" in err_type.lower() or "waf blocked" in err_type.lower():
+                    err_type = "cf-mitigated (403202)"
+
+            ProxyManager.Instance().mark_proxy_result(
+                proxy=active_plg_cfg.proxy.model_dump(),
+                plugin_name=self.m_cfg.plugin_config.plugin_name,
+                action=action,
+                success=False,
+                http_code=biz_err.http_status,
+                error_type=err_type
+            )
+
     def _is_within_active_hours(self) -> bool:
         """判断当前是否在允许创建实例的时间段内"""
         start_str = self.m_cfg.active_time_start
@@ -426,7 +456,11 @@ class BuiltinBookerGCO(BaseBookerGCO):
                 plg_cfg.account = type(plg_cfg.account)(**acc)
 
             if self.m_cfg.need_proxy:
-                proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
+                proxy = ProxyManager.Instance().get_next_proxy(
+                    self.m_cfg.proxy_pool,
+                    self.m_cfg.proxy_cd,
+                    plugin_name=self.m_cfg.plugin_config.plugin_name
+                )
                 plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
 
             instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
@@ -448,6 +482,14 @@ class BuiltinBookerGCO(BaseBookerGCO):
                 self.m_tasks.append(task)
 
             success = True
+            if plg_cfg.proxy.port > 0:
+                ProxyManager.Instance().mark_proxy_result(
+                    proxy=plg_cfg.proxy.model_dump(),
+                    plugin_name=self.m_cfg.plugin_config.plugin_name,
+                    action="spawn",
+                    success=True,
+                    http_code=200
+                )
             self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
         except Exception as e:
             self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
@@ -641,7 +683,11 @@ class OrderBookerGCO(BaseBookerGCO):
             
             acceptable_keys = [target_routing_key]
             if self.m_cfg.need_proxy:
-                proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
+                proxy = ProxyManager.Instance().get_next_proxy(
+                    self.m_cfg.proxy_pool,
+                    self.m_cfg.proxy_cd,
+                    plugin_name=self.m_cfg.plugin_config.plugin_name
+                )
                 plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
 
             instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
@@ -662,6 +708,14 @@ class OrderBookerGCO(BaseBookerGCO):
                 self.m_tasks.append(task)
 
             success = True             
+            if plg_cfg.proxy.port > 0:
+                ProxyManager.Instance().mark_proxy_result(
+                    proxy=plg_cfg.proxy.model_dump(),
+                    plugin_name=self.m_cfg.plugin_config.plugin_name,
+                    action="spawn",
+                    success=True,
+                    http_code=200
+                )
             self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
         except Exception as e:
             self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)

+ 1 - 1
plugins/vfs_plugin.py

@@ -524,7 +524,7 @@ class VfsPlugin(IPlugin):
             raise RateLimiteddError(f"429 Rate Limit: {resp.text[:100]}")
             
         elif resp.status_code == 0:
-            raise BizLogicError(f"Network Error (Fetch Failed): {resp.text}")
+            raise PermissionDeniedError(f"Network Error (Fetch Failed / WAF Blocked): {resp.text}")
             
         else:
             if url.endswith("/login") and resp.status_code == 400:

+ 64 - 13
sentinel.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from typing import List, Dict, Callable, Optional
 
 from core_types import (
-    GroupConfig, PluginConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
+    GroupConfig, PluginConfig, Account, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str,
     BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
     NetworkTimeoutError
 )
@@ -133,9 +133,12 @@ class SentinelGCO:
 
     def _normalize_error(self, err: Exception) -> BizException:
         """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
+        err_str = str(err)
+        # 如果是底层网络/CORS预检被拒导致的 Fetch 失败或 WAF 阻断,精准提升为 PermissionDeniedError
+        if "Failed to fetch" in err_str or "Status: 0" in err_str or "WAF Blocked" in err_str or "cf-mitigated" in err_str:
+            return PermissionDeniedError(message=err_str)
         if isinstance(err, BizException):
             return err
-        err_str = str(err)
         if "42901" in err_str or "Rate limited" in err_str:
             return RateLimiteddError(message=err_str)
         if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
@@ -147,6 +150,15 @@ class SentinelGCO:
             message=f"Unhandled ({type(err).__name__}): {err_str}",
         )
 
+    def _release_account_safely(self, acc: Optional[Account], reason: str = "spawn failed") -> None:
+        """安全归还账号:将云端冷却时间重置为 0,避免非账号问题导致账号被误锁 3 小时"""
+        if acc and acc.id > 0 and acc.username not in ("", "Guest"):
+            try:
+                CloudApi.Instance().lock_account(acc.id, 0)
+                self._log(f"🔓 [Account Released] Account [{acc.username}] immediately released back to pool (Reason: {reason}).")
+            except Exception as e:
+                self._log(f"Failed to release account [{acc.username}]: {e}")
+
     def _dispatch_error_policy(
         self,
         action: str,
@@ -160,22 +172,19 @@ class SentinelGCO:
         - spawn/login 时 429:判定为账号级限流,锁定账号并设置 m_spawn_cooldown_until 退避;
         - query 时 429:会话已登录,判定为出口 IP 频率超限,不锁定账号,移除任务以触发 creator 换代理生成新实例;
         - session expired / 401:移除故障任务;
-        - 403 Forbidden / WAF:判定为出口 IP 被风控,移除任务以轮换代理;
+        - 403 Forbidden / WAF:判定为出口 IP 被风控,移除任务以轮换代理;在 spawn 时立即安全释放账号;
         - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
         """
         biz_err = self._normalize_error(err)
-        acc = None
-        if plg_cfg and plg_cfg.account:
-            acc = plg_cfg.account
-        elif task and getattr(task.instance, "config", None) and task.instance.config.account:
-            acc = task.instance.config.account
+        active_plg_cfg = plg_cfg or (task.instance.config if task else None)
+        acc = active_plg_cfg.account if active_plg_cfg else None
 
         # 1. 目标站限流 / 429
         if isinstance(biz_err, RateLimiteddError):
             if action in ("spawn", "login", "create_session"):
                 backoff = float(self.m_cfg.login_backoff)
                 self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
-                if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0):
+                if acc and acc.username not in ("", "Guest") and acc.id > 0:
                     try:
                         CloudApi.Instance().lock_account(acc.id, int(backoff))
                         self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
@@ -206,11 +215,12 @@ class SentinelGCO:
 
         # 3. 权限被拒 / WAF 拦截 / 403
         elif isinstance(biz_err, PermissionDeniedError):
-            proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)
+            proxy_cd = float(self.m_cfg.proxy_cd or 60.0)
             self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
             if task:
                 self._remove_task(task, f"{action} waf blocked")
-            if action == "spawn":
+            if action in ("spawn", "login", "create_session"):
+                self._release_account_safely(acc, reason="WAF Blocked")
                 self.m_spawn_cooldown_until = time.time() + proxy_cd
 
         # 4. 网络或浏览器超时
@@ -218,15 +228,35 @@ class SentinelGCO:
             self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
             if task:
                 task.next_run = time.time() + 5.0
+            if action in ("spawn", "login", "create_session"):
+                self._release_account_safely(acc, reason="Network Timeout")
+                self.m_spawn_cooldown_until = time.time() + 10.0
 
         # 5. 兜底策略
         else:
             self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
-            if action == "spawn":
+            if action in ("spawn", "login", "create_session"):
+                self._release_account_safely(acc, reason=f"Unexpected error: {biz_err.message}")
                 self.m_spawn_cooldown_until = time.time() + 10.0
             elif task:
                 task.next_run = time.time() + 10.0
 
+        # 6. 统一代理健康指标埋点
+        if active_plg_cfg and isinstance(active_plg_cfg.proxy.port, int) and active_plg_cfg.proxy.port > 0:
+            err_type = biz_err.message
+            if isinstance(biz_err, PermissionDeniedError):
+                if "403202" in err_type or "cf-mitigated" in err_type.lower() or "waf blocked" in err_type.lower():
+                    err_type = "cf-mitigated (403202)"
+
+            ProxyManager.Instance().mark_proxy_result(
+                proxy=active_plg_cfg.proxy.model_dump(),
+                plugin_name=self.m_cfg.plugin_config.plugin_name,
+                action=action,
+                success=False,
+                http_code=biz_err.http_status,
+                error_type=err_type
+            )
+
     def _is_within_active_hours(self) -> bool:
         """
         判断当前是否在允许创建实例的时间段内
@@ -318,6 +348,15 @@ class SentinelGCO:
                             redis_key = self._get_redis_key(apt_type.routing_key)
                             self.redis_client.setex(redis_key, ttl, json.dumps(payload))
                         task.next_run = time.time() + interval
+                        # 埋点上报 query 成功
+                        if task.instance.config.proxy.port > 0:
+                            ProxyManager.Instance().mark_proxy_result(
+                                proxy=task.instance.config.proxy.model_dump(),
+                                plugin_name=self.m_cfg.plugin_config.plugin_name,
+                                action="query",
+                                success=True,
+                                http_code=200
+                            )
                     except Exception as e:
                         self._dispatch_error_policy("query", e, task=task)
                     break
@@ -359,7 +398,11 @@ class SentinelGCO:
                 plg_cfg.account = type(plg_cfg.account)(**acc)
             
             if self.m_cfg.need_proxy:
-                proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
+                proxy = ProxyManager.Instance().get_next_proxy(
+                    self.m_cfg.proxy_pool,
+                    self.m_cfg.proxy_cd,
+                    plugin_name=self.m_cfg.plugin_config.plugin_name
+                )
                 plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
 
             instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
@@ -372,6 +415,14 @@ class SentinelGCO:
             with self.m_lock:
                 self.m_tasks.append(task)
             success = True
+            if plg_cfg.proxy.port > 0:
+                ProxyManager.Instance().mark_proxy_result(
+                    proxy=plg_cfg.proxy.model_dump(),
+                    plugin_name=self.m_cfg.plugin_config.plugin_name,
+                    action="spawn",
+                    success=True,
+                    http_code=200
+                )
             self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
         except Exception as e:
             self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)

+ 18 - 0
test/test_metadata_exception.py

@@ -161,11 +161,23 @@ class TestMetadataExceptions(unittest.TestCase):
 
         # 场景 4: 模拟 PermissionDeniedError
         mock_inst2 = MagicMock()
+        mock_inst2.config = plg_cfg
         task2 = Task(instance=mock_inst2)
         gco.m_tasks.append(task2)
         gco._dispatch_error_policy("query", PermissionDeniedError(), task=task2)
         self.assertNotIn(task2, gco.m_tasks)
 
+        # 场景 5: spawn 触发 PermissionDeniedError (如 WAF 拦截或 Status: 0 / Failed to fetch) -> 立即释放账号(lock=0),并按 proxy_cd 冷却
+        mock_cloud.reset_mock()
+        gco._dispatch_error_policy("spawn", PermissionDeniedError("Network Error (Fetch Failed / WAF Blocked)"), plg_cfg=plg_cfg)
+        mock_cloud.lock_account.assert_called_once_with(999, 0)
+        self.assertGreater(gco.m_spawn_cooldown_until, time.time())
+
+        # 场景 6: spawn 触发原始字符串包含 Failed to fetch -> 自动规整并安全释放账号
+        mock_cloud.reset_mock()
+        gco._dispatch_error_policy("spawn", Exception("TypeError: Failed to fetch"), plg_cfg=plg_cfg)
+        mock_cloud.lock_account.assert_called_once_with(999, 0)
+
     @patch("toolkit.cloud_api.CloudApi.Instance")
     def test_booker_error_dispatch(self, mock_cloud_instance):
         from booker import BuiltinBookerGCO
@@ -201,6 +213,12 @@ class TestMetadataExceptions(unittest.TestCase):
         mock_cloud.lock_account.assert_called_once_with(777, 900)
         self.assertGreater(gco.m_spawn_cooldown_until, time.time())
 
+        # 场景 3: 模拟 spawn 触发 PermissionDeniedError (WAF / Status: 0) -> 立即归还账号(lock=0)
+        mock_cloud.reset_mock()
+        gco._dispatch_error_policy("spawn", PermissionDeniedError("WAF Blocked"), plg_cfg=plg_cfg)
+        mock_cloud.lock_account.assert_called_once_with(777, 0)
+        self.assertGreater(gco.m_spawn_cooldown_until, time.time())
+
 
 if __name__ == "__main__":
     unittest.main()

+ 97 - 4
test/test_proxy_manager.py

@@ -1,6 +1,8 @@
 import os
 import sys
+import time
 import unittest
+from unittest.mock import MagicMock, patch
 
 project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 if project_root not in sys.path:
@@ -10,9 +12,28 @@ from toolkit.proxy_manager import ProxyManager
 from core_types import Proxy, BizLogicError
 
 
+from toolkit.cloud_api import CloudApi
+
+
 class TestProxyManager(unittest.TestCase):
     def setUp(self):
         self.pm = ProxyManager.Instance()
+        # 清理可能存在的隔离状态
+        self.pm._quarantine.clear()
+        self.pm._sliding_history.clear()
+        self.pm._reputation_cache.clear()
+        # Mock 掉耗时的真实外部探测与云端 HTTP 调用
+        self.patcher_probe = patch.object(self.pm, "_async_probe_egress_ip", return_value=None)
+        self.patcher_probe.start()
+        self.patcher_rep = patch.object(CloudApi.Instance(), "get_proxy_reputation_map", return_value={})
+        self.patcher_rep.start()
+        self.patcher_report = patch.object(CloudApi.Instance(), "report_proxy_metric", return_value=True)
+        self.patcher_report.start()
+
+    def tearDown(self):
+        self.patcher_probe.stop()
+        self.patcher_rep.stop()
+        self.patcher_report.stop()
 
     def test_local_proxy_single(self):
         # 测试 local 池 (单端口)
@@ -20,18 +41,17 @@ class TestProxyManager(unittest.TestCase):
         self.assertIsInstance(res, dict)
         self.assertEqual(res["ip"], "127.0.0.1")
         self.assertEqual(res["port"], 7897)
-        # 验证能够无缝构建 Proxy
         proxy_obj = Proxy(**res)
         self.assertEqual(proxy_obj.port, 7897)
 
     def test_local_proxy_multi_pool(self):
-        # 测试 local_multi_pool (多端口 10001 ~ 10306)
+        # 测试 local_multi_pool
         res = self.pm.get_next_proxy(["local:local_multi_pool"])
         self.assertIsInstance(res, dict)
         self.assertEqual(res["ip"], "127.0.0.1")
-        self.assertTrue(10001 <= res["port"] <= 10400)
+        self.assertTrue(res["port"] >= 10000)
         proxy_obj = Proxy(**res)
-        self.assertTrue(10001 <= proxy_obj.port <= 10400)
+        self.assertTrue(proxy_obj.port >= 10000)
 
     def test_invalid_prefix(self):
         # 测试未带前缀的非法格式,必须抛出 BizLogicError
@@ -45,6 +65,79 @@ class TestProxyManager(unittest.TestCase):
             self.pm.get_next_proxy(["local:non_existent_pool_12345"])
         self.assertIn("No proxies found in local pools", str(ctx.exception))
 
+    def test_reputation_quarantine_and_revival(self):
+        """测试专属 WAF 阻断进入隔离名单,以及成功后自愈复活"""
+        plugin = "vfs_plugin"
+        proxy = self.pm.get_next_proxy(["local:local_multi_pool"], plugin_name=plugin)
+        port = proxy["port"]
+        egress_ip = proxy["egress_ip"]
+
+        # 模拟一次 403202 cf-mitigated 专属阻断
+        self.pm.mark_proxy_result(
+            proxy=proxy,
+            plugin_name=plugin,
+            action="spawn",
+            success=False,
+            http_code=403,
+            error_type="cf-mitigated (403202)"
+        )
+
+        # 验证该 IP 已进入隔离名单
+        self.assertIn(egress_ip, self.pm._quarantine.get(plugin, {}))
+
+        # 模拟下一次该 IP 执行成功,验证自愈复活
+        self.pm.mark_proxy_result(
+            proxy=proxy,
+            plugin_name=plugin,
+            action="spawn",
+            success=True,
+            http_code=200
+        )
+        self.assertNotIn(egress_ip, self.pm._quarantine.get(plugin, {}))
+
+    def test_site_outage_freeze(self):
+        """测试 Layer 1: 全站故障时冻结扣分与隔离"""
+        plugin = "vfs_plugin"
+        # 构造跨多个不同 IP 的连续系统级故障 (502 / 503)
+        for i in range(6):
+            fake_proxy = {"port": 16000 + i, "egress_ip": f"104.28.1.{i}", "name": f"node_{i}"}
+            self.pm.mark_proxy_result(
+                proxy=fake_proxy,
+                plugin_name=plugin,
+                action="spawn",
+                success=False,
+                http_code=502,
+                error_type="Bad Gateway"
+            )
+
+        # 此时全站故障已触发
+        self.assertTrue(self.pm._is_site_outage(plugin))
+
+        # 在全站故障状态下,即使有节点发生 WAF 阻断,也不会被拉黑隔离
+        new_proxy = {"port": 16099, "egress_ip": "104.28.99.99", "name": "node_99"}
+        self.pm.mark_proxy_result(
+            proxy=new_proxy,
+            plugin_name=plugin,
+            action="spawn",
+            success=False,
+            http_code=403,
+            error_type="cf-mitigated (403202)"
+        )
+        self.assertNotIn("104.28.99.99", self.pm._quarantine.get(plugin, {}))
+
+    def test_minimum_viable_pool_anti_starvation(self):
+        """测试 Layer 3: 最低可用保底池防雪崩,过度拉黑时强制重置隔离名单"""
+        plugin = "test_plugin"
+        # 人为把隔离名单填满
+        self.pm._quarantine[plugin] = {
+            f"port_{16000 + i}": time.time() + 1800 for i in range(1000)
+        }
+
+        # 获取下一个代理,不应该抛出异常,应该触发防雪崩重置并成功返回代理
+        proxy = self.pm.get_next_proxy(["local:local_multi_pool"], plugin_name=plugin)
+        self.assertIsInstance(proxy, dict)
+        self.assertTrue(proxy["port"] >= 10000)
+
 
 if __name__ == "__main__":
     unittest.main()

+ 34 - 0
toolkit/cloud_api.py

@@ -348,6 +348,40 @@ class CloudApi:
         else:
             raise BizLogicError(message=f"Get next account biz error: {result.get('message')}")
 
+    def report_proxy_metric(self, metric: Dict[str, Any]) -> bool:
+        """
+        上报代理指标埋点数据 (Fail-Silent)
+        @param metric: 包含 client_id, plugin_name, egress_ip, local_port, node_name, action, success, http_code, error_type, latency_ms 等
+        @return: 是否上报成功
+        """
+        try:
+            url = f"{self.base_url}/api/proxy/telemetry"
+            headers = self._get_headers()
+            resp = self._perform_request("POST", url, headers=headers, json_data=metric, timeout=5)
+            return resp.status_code == 200
+        except Exception as e:
+            VSC_DEBUG("vs_cloud", f"[report_proxy_metric] Failed to report proxy telemetry: {e}")
+            return False
+
+    def get_proxy_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
+        """
+        从云端拉取指定插件的代理信誉字典 (Fail-Silent)
+        @param plugin_name: 插件名称,如 'vfs_plugin'
+        @return: 字典格式 { "egress_ip": { "score": 90, "state": "trusted", ... } }
+        """
+        try:
+            url = f"{self.base_url}/api/proxy/reputation/{quote(plugin_name)}"
+            headers = self._get_headers()
+            resp = self._perform_request("GET", url, headers=headers, timeout=5)
+            result = resp.json()
+            if result.get("code") == 0:
+                data = result.get("data") or {}
+                return data.get("reputation_map", {})
+            return {}
+        except Exception as e:
+            VSC_DEBUG("vs_cloud", f"[get_proxy_reputation_map] Failed to fetch proxy reputation for {plugin_name}: {e}")
+            return {}
+
     def slot_snapshot_report(self, query_payload: Dict[str, Any] = {}):
         url = f"{self.base_url}/api/slots/report"
         headers = self._get_headers()

+ 350 - 24
toolkit/proxy_manager.py

@@ -1,22 +1,29 @@
 import os
+import time
 import json
 import yaml
 import random
+import threading
+import collections
+import concurrent.futures
 from typing import List, Dict, Any, Optional
 
+import requests
 import configure
 from toolkit.cloud_api import CloudApi
 from core_types import BizLogicError
-from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR
+from log_macros import VSC_INFO, VSC_WARN, VSC_ERROR, VSC_DEBUG
 
 
 class ProxyManager:
     """
-    统一代理调度管理器。
-    支持显式前缀区分:
-      - "local:<pool_name>"  -> 从本地 config/proxies.yaml 随机分配
-      - "cloud:<pool_name>"  -> 通过 CloudApi 请求云端代理池
-    严格校验:未带合法前缀将抛出异常。
+    统一代理调度与信誉管理中心。
+    支持:
+      - 显式前缀区分:"local:<pool_name>" / "cloud:<pool_name>"
+      - 出口 IP 探针与缓存机制 (Egress Resolver & Cache)
+      - 本地通道与云端信誉库求交加权调度 (85% 优选 + 15% 探索)
+      - 五层防雪崩护栏 (全站故障冻结、20% 最低保底池、半开自愈、底层硬兜底)
+      - 异步指标埋点与上报 (Fail-Silent)
     """
     _instance = None
 
@@ -34,6 +41,23 @@ class ProxyManager:
         self.project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         self.local_proxies_file = os.path.join(self.project_root, "config", "proxies.yaml")
 
+        self._lock = threading.RLock()
+
+        # 端口 -> 公网出口 IP 缓存与节点名称缓存
+        self._port_to_egress: Dict[int, str] = {}
+        self._port_to_name: Dict[int, str] = {}
+        self._probing_ports = set()
+
+        # 插件维度本地临时隔离黑名单: { plugin_name: { egress_ip: expire_ts } }
+        self._quarantine: Dict[str, Dict[str, float]] = {}
+
+        # 插件维度云端信誉缓存: { plugin_name: { "last_fetch": float, "map": { egress_ip: {...} } } }
+        self._reputation_cache: Dict[str, Dict[str, Any]] = {}
+        self._fetching_reputation = set()
+
+        # 插件维度滑动调用历史 (用于全站故障判定): { plugin_name: collections.deque(maxlen=30) }
+        self._sliding_history: Dict[str, collections.deque] = {}
+
     def _load_local_proxies(self) -> Dict[str, List[Dict[str, Any]]]:
         if not os.path.exists(self.local_proxies_file):
             raise BizLogicError(message=f"Local proxy config file not found: {self.local_proxies_file}")
@@ -43,14 +67,147 @@ class ProxyManager:
         except Exception as e:
             raise BizLogicError(message=f"Failed to parse {self.local_proxies_file}: {e}")
 
-    def get_next_proxy(self, pools: List[str], proxy_cd: int = 60) -> Dict[str, Any]:
+    # ==========================================
+    # 出口 IP 探针与缓存 (Egress Resolver)
+    # ==========================================
+
+    def get_cached_egress_ip(self, port: int) -> Optional[str]:
+        with self._lock:
+            return self._port_to_egress.get(port)
+
+    def invalidate_egress_cache(self, port: int):
+        """当节点疑似漂移时清除缓存"""
+        with self._lock:
+            if port in self._port_to_egress:
+                old_ip = self._port_to_egress.pop(port, None)
+                VSC_DEBUG("vs_proxy", f"[invalidate_egress_cache] Invalidated port {port} (old egress: {old_ip})")
+
+    def _async_probe_egress_ip(self, port: int, proto: str = "http"):
+        """后台异步探测端口的真实公网出口 IP,不阻塞主调度"""
+        with self._lock:
+            if port in self._port_to_egress or port in self._probing_ports:
+                return
+            self._probing_ports.add(port)
+
+        def _do_probe():
+            try:
+                proxy_url = f"{proto}://127.0.0.1:{port}"
+                proxies = {"http": proxy_url, "https": proxy_url}
+                # 尝试轻量公网 IP 端点
+                resp = requests.get("http://api.ipify.org", proxies=proxies, timeout=2.5)
+                if resp.status_code == 200 and resp.text.strip():
+                    ip = resp.text.strip()
+                    with self._lock:
+                        self._port_to_egress[port] = ip
+                    VSC_DEBUG("vs_proxy", f"[probe_egress_ip] Resolved port {port} -> {ip}")
+            except Exception:
+                pass
+            finally:
+                with self._lock:
+                    self._probing_ports.discard(port)
+
+        t = threading.Thread(target=_do_probe, daemon=True)
+        t.start()
+
+    def _resolve_ip_identifier(self, node: Dict[str, Any], trigger_probe: bool = False) -> str:
+        """获取节点的 IP 标识符(优先真实出口 IP,未解析则使用端口锚点)"""
+        port = int(node.get("port", 0))
+        configured_egress = node.get("egress_ip")
+        if configured_egress:
+            return configured_egress
+
+        with self._lock:
+            cached = self._port_to_egress.get(port)
+            if cached:
+                return cached
+
+        if trigger_probe and port:
+            proto = node.get("proto", "http")
+            self._async_probe_egress_ip(port, proto)
+
+        return f"port_{port}"
+
+    # ==========================================
+    # 云端信誉同步与防雪崩护栏
+    # ==========================================
+
+    def _get_reputation_map(self, plugin_name: str) -> Dict[str, Dict[str, Any]]:
+        """获取云端信誉字典 (绝不同步阻塞主流程,后台异步刷新)"""
+        now = time.time()
+        with self._lock:
+            cache_entry = self._reputation_cache.get(plugin_name)
+            if cache_entry and (now - cache_entry.get("last_fetch", 0) < 300):
+                return cache_entry.get("map", {})
+
+            if plugin_name in self._fetching_reputation:
+                return (cache_entry.get("map", {}) if cache_entry else {})
+
+            self._fetching_reputation.add(plugin_name)
+
+        def _fetch_worker():
+            try:
+                rep_map = CloudApi.Instance().get_proxy_reputation_map(plugin_name)
+                with self._lock:
+                    self._reputation_cache[plugin_name] = {
+                        "last_fetch": time.time(),
+                        "map": rep_map or {}
+                    }
+            except Exception:
+                pass
+            finally:
+                with self._lock:
+                    self._fetching_reputation.discard(plugin_name)
+
+        threading.Thread(target=_fetch_worker, daemon=True).start()
+        return (cache_entry.get("map", {}) if cache_entry else {})
+
+    def _is_site_outage(self, plugin_name: str) -> bool:
+        """
+        Layer 1 防护:全站故障判决
+        过去 180s 内,跨至少 2 个不同 IP 且样本 >= 5,若失败率 >= 90% 或均报 502/503/520~526,判定为全站故障
+        """
+        with self._lock:
+            history = self._sliding_history.get(plugin_name)
+            if not history or len(history) < 5:
+                return False
+
+            now = time.time()
+            recent_events = [e for e in history if now - e["ts"] < 180]
+            if len(recent_events) < 5:
+                return False
+
+            distinct_ips = {e["egress_ip"] for e in recent_events}
+            if len(distinct_ips) < 2:
+                return False
+
+            failures = [e for e in recent_events if not e["success"]]
+            fail_rate = len(failures) / len(recent_events)
+
+            # 严重系统级错误数量
+            server_errors = [e for e in failures if e["http_code"] in (500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526)]
+
+            if fail_rate >= 0.90 or (len(server_errors) / len(recent_events) >= 0.70):
+                return True
+
+            return False
+
+    # ==========================================
+    # 核心调度逻辑 (求交加权 + 五层护栏)
+    # ==========================================
+
+    def get_next_proxy(
+        self,
+        pools: List[str],
+        proxy_cd: int = 60,
+        plugin_name: Optional[str] = None
+    ) -> Dict[str, Any]:
         """
         根据 pools 列表解析并获取下一个代理。
         @param pools: 代理池列表,例如 ["local:local_multi_pool"] 或 ["cloud:proxy-cheap"]
         @param proxy_cd: 代理冷却时间(云端使用)
-        @return dict 格式标准代理信息 (包含 id, proto, ip, port, username, password)
+        @param plugin_name: 插件名称(如 'vfs_plugin'),用于结合信誉分和隔离策略
+        @return dict 格式标准代理信息 (包含 id, proto, ip, port, username, password, egress_ip, name)
         """
-        # 如果配置了全局静态测试代理则优先使用
         if getattr(configure, "TEST_PROXY", None):
             return configure.TEST_PROXY
 
@@ -77,7 +234,6 @@ class ProxyManager:
                     message=f"Invalid proxy pool format '{pool_item}'. Must start with 'local:' or 'cloud:'."
                 )
 
-        # 优先支持本地代理池
         if local_targets:
             all_local_data = self._load_local_proxies()
             nodes_list = all_local_data.get("nodes") or all_local_data.get("proxies") or []
@@ -87,7 +243,7 @@ class ProxyManager:
             pools_dict = all_local_data.get("pools", {}) if "pools" in all_local_data else all_local_data
 
             candidate_proxies: List[Dict[str, Any]] = []
-            
+
             for lt in local_targets:
                 pool_items = pools_dict.get(lt, [])
                 if pool_items and isinstance(pool_items, list):
@@ -104,21 +260,191 @@ class ProxyManager:
                     message=f"No proxies found in local pools: {local_targets} from {self.local_proxies_file}"
                 )
 
-            chosen = random.choice(candidate_proxies)
-            
-            # 规范化代理字典,确保与 Proxy 字段完全契合
-            proxy_dict = {
-                "id": chosen.get("id", 0),
-                "proto": chosen.get("proto", "http"),
-                "ip": chosen.get("ip", "127.0.0.1"),
-                "port": int(chosen.get("port", 80)),
-                "username": chosen.get("username", ""),
-                "password": chosen.get("password", "")
-            }
-            return proxy_dict
+            # 记录节点名称缓存
+            for node in candidate_proxies:
+                port = int(node.get("port", 0))
+                if port and "name" in node:
+                    self._port_to_name[port] = node["name"]
+
+            # 如果没有指定 plugin_name,退化为经典随机轮询
+            if not plugin_name:
+                chosen = random.choice(candidate_proxies)
+                return self._format_proxy_dict(chosen)
+
+            # ------------------------------------------------------------------
+            # 智能信誉分级调度 (求交 + 梯队路由 + 五层护栏)
+            # ------------------------------------------------------------------
+            now = time.time()
+            rep_map = self._get_reputation_map(plugin_name)
+
+            with self._lock:
+                quarantine_map = self._quarantine.setdefault(plugin_name, {})
+                # 清理过期隔离节点
+                expired_ips = [ip for ip, exp in quarantine_map.items() if now > exp]
+                for ip in expired_ips:
+                    quarantine_map.pop(ip, None)
+
+                # Layer 3 防护:最低可用保底池检查 (保底 20% 或至少 5 个可用节点)
+                total_nodes_count = len(candidate_proxies)
+                quarantined_count = sum(
+                    1 for node in candidate_proxies
+                    if self._resolve_ip_identifier(node) in quarantine_map
+                )
+
+                # 如果隔离比例超 70% 或剩余可用节点 < 20% (且小于 5),触发熔断重置
+                if total_nodes_count > 0:
+                    available_ratio = (total_nodes_count - quarantined_count) / total_nodes_count
+                    if available_ratio < 0.20 or (total_nodes_count - quarantined_count) < min(5, total_nodes_count):
+                        VSC_WARN("vs_proxy", f"【代理防雪崩】插件 {plugin_name} 可用节点低于保底阈值(可用率 {available_ratio:.1%}),强制清空临时隔离名单!")
+                        quarantine_map.clear()
+
+            # 划分三梯队
+            trusted_tier: List[Dict[str, Any]] = []
+            explore_tier: List[Dict[str, Any]] = []
+
+            for node in candidate_proxies:
+                ip_id = self._resolve_ip_identifier(node)
+                # 检查是否在隔离名单中
+                if ip_id in quarantine_map:
+                    continue
+
+                # 检查云端信誉
+                rep_info = rep_map.get(ip_id)
+                if rep_info and rep_info.get("score", 0) >= 75 and rep_info.get("state") == "trusted":
+                    trusted_tier.append(node)
+                else:
+                    explore_tier.append(node)
+
+            # 梯队加权调度:85% 优选,15% 探索
+            chosen = None
+            if trusted_tier and explore_tier:
+                if random.random() < 0.85:
+                    chosen = random.choice(trusted_tier)
+                else:
+                    chosen = random.choice(explore_tier)
+            elif trusted_tier:
+                chosen = random.choice(trusted_tier)
+            elif explore_tier:
+                chosen = random.choice(explore_tier)
+
+            # Layer 5 防护:本地代码硬保底 (如果过滤后为空,回退全池随机)
+            if not chosen:
+                VSC_WARN("vs_proxy", f"【代理硬保底】插件 {plugin_name} 经过信誉过滤后无可用节点,触发全量随机兜底!")
+                chosen = random.choice(candidate_proxies)
+
+            return self._format_proxy_dict(chosen)
 
-        # 如果指定的是云端代理池
         if cloud_targets:
             return CloudApi.Instance().get_next_proxy(cloud_targets, proxy_cd=proxy_cd)
 
         raise BizLogicError(message=f"No valid local or cloud proxy pool specified in {pools}")
+
+    def _format_proxy_dict(self, chosen: Dict[str, Any]) -> Dict[str, Any]:
+        """规范化代理字典"""
+        port = int(chosen.get("port", 80))
+        egress_ip = self._resolve_ip_identifier(chosen, trigger_probe=True)
+        return {
+            "id": chosen.get("id", 0),
+            "proto": chosen.get("proto", "http"),
+            "ip": chosen.get("ip", "127.0.0.1"),
+            "port": port,
+            "username": chosen.get("username", ""),
+            "password": chosen.get("password", ""),
+            "name": chosen.get("name", self._port_to_name.get(port, "")),
+            "egress_ip": egress_ip,
+        }
+
+    # ==========================================
+    # 指标埋点打桩与实时反馈 (Telemetry Upstream)
+    # ==========================================
+
+    def mark_proxy_result(
+        self,
+        proxy: Dict[str, Any],
+        plugin_name: str,
+        action: str,
+        success: bool,
+        http_code: int = 200,
+        error_type: str = "",
+        latency_ms: int = 0
+    ):
+        """
+        标记代理使用结果并异步上报。
+        内置:
+          - 全站故障判决 (Site Outage)
+          - 针对性阻断识别与临时隔离 (Quarantine)
+          - 半开自愈复活 (Half-Open Revival)
+          - 漂移检测 (Drift Detection)
+        """
+        if not proxy or not plugin_name:
+            return
+
+        port = int(proxy.get("port", 0))
+        egress_ip = proxy.get("egress_ip") or self._resolve_ip_identifier(proxy)
+        node_name = proxy.get("name") or self._port_to_name.get(port, "")
+
+        # 1. 记入滑动调用历史
+        now = time.time()
+        with self._lock:
+            history = self._sliding_history.setdefault(plugin_name, collections.deque(maxlen=30))
+            history.append({
+                "ts": now,
+                "egress_ip": egress_ip,
+                "success": success,
+                "http_code": http_code,
+                "error_type": error_type
+            })
+
+        # 2. 判断是否处于全站故障期
+        is_outage = self._is_site_outage(plugin_name)
+
+        with self._lock:
+            quarantine_map = self._quarantine.setdefault(plugin_name, {})
+
+            if success:
+                # 成功事件:若在隔离期或半开期,立刻解除隔离并复活
+                if egress_ip in quarantine_map:
+                    quarantine_map.pop(egress_ip, None)
+                    VSC_INFO("vs_proxy", f"【代理自愈复活】出口 IP {egress_ip} (Port {port}) 在插件 {plugin_name} 验证成功,已移出隔离名单!")
+            else:
+                # 失败事件
+                if is_outage:
+                    VSC_WARN("vs_proxy", f"【全站故障保护】目标网站疑似出现全站级故障,已冻结对出口 IP {egress_ip} 的扣分与隔离!")
+                else:
+                    # Layer 2: 错误精细分类判定
+                    is_waf_block = (
+                        "cf-mitigated" in error_type.lower() or
+                        "403202" in error_type or
+                        "waf blocked" in error_type.lower() or
+                        (http_code == 403 and "mitigated" in error_type.lower())
+                    )
+                    if is_waf_block:
+                        # 针对性阻断:进入隔离名单(默认 1800 秒)
+                        quarantine_map[egress_ip] = now + 1800
+                        VSC_WARN("vs_proxy", f"【节点临时隔离】出口 IP {egress_ip} (Port {port}) 触发针对性 WAF 阻断,临时隔离 30 分钟!")
+                        # 触发潜在漂移检测:清除缓存以便重新探查真实公网 IP 是否已变
+                        self.invalidate_egress_cache(port)
+
+        # 3. 异步上报埋点指标至云端 (Fail-Silent)
+        metric_payload = {
+            "client_id": getattr(configure, "NODE_ID", "local_node"),
+            "plugin_name": plugin_name,
+            "local_port": port,
+            "node_name": node_name,
+            "egress_ip": egress_ip,
+            "action": action,
+            "success": success,
+            "http_code": http_code,
+            "error_type": error_type,
+            "latency_ms": latency_ms,
+            "timestamp": int(now)
+        }
+        
+        def _report_worker():
+            try:
+                CloudApi.Instance().report_proxy_metric(metric_payload)
+            except Exception:
+                pass
+
+        t = threading.Thread(target=_report_worker, daemon=True)
+        t.start()