proxy_reputation_service.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. import time
  2. from datetime import datetime, timedelta
  3. from typing import Optional, Dict, Any, Tuple
  4. from sqlalchemy.ext.asyncio import AsyncSession
  5. from sqlalchemy import select, func, and_
  6. from app.models.proxy_reputation import ProxyReputation, ProxyTelemetryLog
  7. from app.schemas.proxy_reputation import (
  8. ProxyTelemetryReportRequest,
  9. ProxyReputationMapResponse,
  10. ProxyReputationItemOut,
  11. )
  12. from app.core.logger import logger
  13. class ProxyReputationService:
  14. """
  15. 云端代理信誉分与指标埋点系统业务服务
  16. 实现:
  17. 1. 代理指标埋点上报及流水日志记录
  18. 2. IP 评分与状态机模型 (trusted / probation / blocked)
  19. 3. 目标站严重系统错误豁免 (500, 502, 503, 504, 520~526)
  20. 4. 黑名单 30 分钟 TTL 与半开自动复活 (Half-Open Revival)
  21. 5. 全网故障防雪崩保护熔断机制 (Global Outage Breaker)
  22. 6. 插件 IP 全景信誉字典查询
  23. """
  24. # 严重系统错误豁免状态码集合(目标站点自身故障,不扣除 IP 分数)
  25. EXEMPT_HTTP_CODES = {500, 502, 503, 504}
  26. CLOUDFLARE_OUTAGE_MIN = 520
  27. CLOUDFLARE_OUTAGE_MAX = 526
  28. # 黑名单封禁 TTL(分钟)
  29. BLOCKED_TTL_MINUTES = 30
  30. @classmethod
  31. def is_exempt_http_code(cls, http_code: int) -> bool:
  32. """
  33. 判断是否属于目标站点自身的严重系统错误(豁免扣分)
  34. """
  35. if http_code in cls.EXEMPT_HTTP_CODES:
  36. return True
  37. if cls.CLOUDFLARE_OUTAGE_MIN <= http_code <= cls.CLOUDFLARE_OUTAGE_MAX:
  38. return True
  39. return False
  40. @classmethod
  41. def is_waf_blocked(cls, http_code: int, error_type: Optional[str]) -> bool:
  42. """
  43. 判断是否命中 Cloudflare WAF 阻断 (403202 / cf-mitigated / WAF Blocked)
  44. """
  45. if not error_type and http_code == 403:
  46. return True
  47. if error_type:
  48. lowered = error_type.lower()
  49. if (
  50. "403202" in lowered
  51. or "cf-mitigated" in lowered
  52. or "waf blocked" in lowered
  53. or "waf" in lowered
  54. or (http_code == 403 and "mitigated" in lowered)
  55. ):
  56. return True
  57. return http_code == 403
  58. @classmethod
  59. async def check_global_outage_breaker(
  60. cls,
  61. db: AsyncSession,
  62. plugin_name: str,
  63. lookback_minutes: int = 3,
  64. min_samples: int = 5,
  65. outage_failure_rate_threshold: float = 0.85,
  66. ) -> bool:
  67. """
  68. 检查目标站全网故障保护 (Global Outage Breaker)
  69. 统计最近 lookback_minutes (默认3分钟) 内该插件的所有上报事件:
  70. 若样本数 >= min_samples 且失败率突发超过 85%,判定处于全网故障期,暂停降分与拉黑。
  71. """
  72. since_time = datetime.utcnow() - timedelta(minutes=lookback_minutes)
  73. stmt = (
  74. select(
  75. func.count(ProxyTelemetryLog.id).label("total_cnt"),
  76. func.sum(
  77. func.case((ProxyTelemetryLog.success.is_(False), 1), else_=0)
  78. ).label("fail_cnt"),
  79. )
  80. .where(
  81. and_(
  82. ProxyTelemetryLog.plugin_name == plugin_name,
  83. ProxyTelemetryLog.created_at >= since_time,
  84. )
  85. )
  86. )
  87. res = await db.execute(stmt)
  88. row = res.one_or_none()
  89. if not row:
  90. return False
  91. total_cnt = row.total_cnt or 0
  92. fail_cnt = row.fail_cnt or 0
  93. if total_cnt >= min_samples:
  94. fail_rate = fail_cnt / total_cnt
  95. if fail_rate > outage_failure_rate_threshold:
  96. logger.warning(
  97. f"🚨 [Global Outage Breaker] 触发全网故障保护: plugin={plugin_name}, "
  98. f"样本数={total_cnt}, 失败数={fail_cnt}, 失败率={fail_rate:.2%}"
  99. )
  100. return True
  101. return False
  102. @classmethod
  103. async def record_telemetry(
  104. cls,
  105. db: AsyncSession,
  106. payload: ProxyTelemetryReportRequest,
  107. ) -> bool:
  108. """
  109. 接收并处理客户端上报的指标数据:
  110. 1. 写入 proxy_telemetry_log 流水表;
  111. 2. 更新/初始化 proxy_reputation 信誉表;
  112. 3. 应用状态机、豁免规则、降级与防雪崩熔断算法。
  113. """
  114. now = datetime.utcnow()
  115. now_ts = payload.timestamp or int(time.time())
  116. # 1. 记录流水日志
  117. log_entry = ProxyTelemetryLog(
  118. client_id=payload.client_id,
  119. plugin_name=payload.plugin_name,
  120. egress_ip=payload.egress_ip,
  121. local_port=payload.local_port,
  122. node_name=payload.node_name,
  123. action=payload.action,
  124. success=payload.success,
  125. http_code=payload.http_code,
  126. error_type=payload.error_type,
  127. latency_ms=payload.latency_ms or 0,
  128. timestamp=now_ts,
  129. created_at=now,
  130. )
  131. db.add(log_entry)
  132. # 2. 检查全网目标故障保护 (Global Outage Breaker)
  133. is_global_outage = await cls.check_global_outage_breaker(db, payload.plugin_name)
  134. # 3. 查询当前 IP 的信誉记录
  135. stmt = select(ProxyReputation).where(
  136. and_(
  137. ProxyReputation.plugin_name == payload.plugin_name,
  138. ProxyReputation.egress_ip == payload.egress_ip,
  139. )
  140. )
  141. res = await db.execute(stmt)
  142. record = res.scalar_one_or_none()
  143. if not record:
  144. # 初始状态:新上报公网 IP 初始分 60 分,状态 probation
  145. record = ProxyReputation(
  146. plugin_name=payload.plugin_name,
  147. egress_ip=payload.egress_ip,
  148. score=60,
  149. state="probation",
  150. total_calls=0,
  151. success_calls=0,
  152. consecutive_successes=0,
  153. consecutive_fails=0,
  154. last_http_code=payload.http_code,
  155. last_error_type=payload.error_type,
  156. last_seen=now_ts,
  157. unblock_time=None,
  158. created_at=now,
  159. updated_at=now,
  160. )
  161. db.add(record)
  162. else:
  163. # 4. 黑名单租期与自动半开复活检查 (TTL & Half-Open Revival)
  164. if record.state == "blocked":
  165. if record.unblock_time and now >= record.unblock_time:
  166. logger.info(
  167. f"🔄 [Half-Open Revival] 代理节点解封复活: plugin={record.plugin_name}, ip={record.egress_ip}"
  168. )
  169. record.state = "probation"
  170. record.score = 50
  171. record.consecutive_successes = 0
  172. record.unblock_time = None
  173. # 更新调用计数与元数据
  174. record.total_calls += 1
  175. record.last_seen = now_ts
  176. record.last_http_code = payload.http_code
  177. record.last_error_type = payload.error_type
  178. record.updated_at = now
  179. # 5. 评分与状态机更新计算
  180. if is_global_outage:
  181. # 全网故障期:暂停所有 IP 的降分与拉黑计算,保持信誉字典冻结
  182. if payload.success:
  183. record.success_calls += 1
  184. # 记录维持不变
  185. logger.info(
  186. f"🛡️ [Breaker Active] 处于全网故障期,保持 IP 字典打分冻结: ip={record.egress_ip}, score={record.score}"
  187. )
  188. else:
  189. if payload.success:
  190. # 成功调用
  191. record.success_calls += 1
  192. record.consecutive_successes += 1
  193. record.consecutive_fails = 0
  194. # 加分规则:一次成功调用 +5 分(上限 100 分)
  195. record.score = min(100, record.score + 5)
  196. # 晋升规则:连续成功 3 次且分数 >= 75 晋升为 trusted
  197. if record.consecutive_successes >= 3 and record.score >= 75:
  198. record.state = "trusted"
  199. record.unblock_time = None
  200. elif record.state == "blocked":
  201. # 试探成功脱离 blocked
  202. record.state = "probation"
  203. record.unblock_time = None
  204. else:
  205. # 失败调用
  206. # 检查严重系统错误豁免
  207. if cls.is_exempt_http_code(payload.http_code):
  208. logger.info(
  209. f"🕊️ [Exempt Error] 目标系统严重故障豁免扣分: http_code={payload.http_code}, ip={record.egress_ip}"
  210. )
  211. # 豁免时不扣分,不累积失败
  212. else:
  213. record.consecutive_fails += 1
  214. record.consecutive_successes = 0
  215. # 判断是否命中 WAF 阻断
  216. if cls.is_waf_blocked(payload.http_code, payload.error_type):
  217. # 直接扣除 40 分
  218. record.score = max(0, record.score - 40)
  219. else:
  220. # 普通业务错误或客户端超时:扣除 2 分
  221. record.score = max(0, record.score - 2)
  222. # 状态机降级逻辑
  223. if record.score < 40:
  224. record.state = "blocked"
  225. record.unblock_time = now + timedelta(minutes=cls.BLOCKED_TTL_MINUTES)
  226. logger.warning(
  227. f"⛔ [Blocked] 代理节点降级拉黑: ip={record.egress_ip}, score={record.score}, "
  228. f"unblock_at={record.unblock_time}"
  229. )
  230. elif record.state == "trusted" and record.score < 75:
  231. record.state = "probation"
  232. await db.commit()
  233. return True
  234. @classmethod
  235. async def get_reputation_map(
  236. cls,
  237. db: AsyncSession,
  238. plugin_name: str,
  239. ) -> ProxyReputationMapResponse:
  240. """
  241. 获取指定插件的代理信誉全景字典
  242. 同时对已达 30 分钟 TTL 的 blocked IP 执行半开自动复活 (Half-Open Revival)
  243. """
  244. now = datetime.utcnow()
  245. stmt = select(ProxyReputation).where(ProxyReputation.plugin_name == plugin_name)
  246. res = await db.execute(stmt)
  247. records = res.scalars().all()
  248. reputation_map: Dict[str, ProxyReputationItemOut] = {}
  249. max_updated_at: Optional[datetime] = None
  250. has_revived = False
  251. for r in records:
  252. # 检查自动复活
  253. if r.state == "blocked" and r.unblock_time and now >= r.unblock_time:
  254. r.state = "probation"
  255. r.score = 50
  256. r.consecutive_successes = 0
  257. r.unblock_time = None
  258. r.updated_at = now
  259. has_revived = True
  260. if max_updated_at is None or (r.updated_at and r.updated_at > max_updated_at):
  261. max_updated_at = r.updated_at
  262. success_rate = (
  263. round(r.success_calls / r.total_calls, 2) if r.total_calls > 0 else 1.0
  264. )
  265. last_seen_ts = r.last_seen
  266. if not last_seen_ts:
  267. last_seen_ts = int(r.updated_at.timestamp()) if r.updated_at else int(time.time())
  268. reputation_map[r.egress_ip] = ProxyReputationItemOut(
  269. score=r.score,
  270. state=r.state,
  271. success_rate=success_rate,
  272. last_seen=last_seen_ts,
  273. )
  274. if has_revived:
  275. await db.commit()
  276. version = int(max_updated_at.timestamp()) if max_updated_at else int(time.time())
  277. return ProxyReputationMapResponse(
  278. plugin_name=plugin_name,
  279. version=version,
  280. reputation_map=reputation_map,
  281. )