| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313 |
- import time
- from datetime import datetime, timedelta
- from typing import Optional, Dict, Any, Tuple
- from sqlalchemy.ext.asyncio import AsyncSession
- from sqlalchemy import select, func, and_
- from app.models.proxy_reputation import ProxyReputation, ProxyTelemetryLog
- from app.schemas.proxy_reputation import (
- ProxyTelemetryReportRequest,
- ProxyReputationMapResponse,
- ProxyReputationItemOut,
- )
- from app.core.logger import logger
- class ProxyReputationService:
- """
- 云端代理信誉分与指标埋点系统业务服务
- 实现:
- 1. 代理指标埋点上报及流水日志记录
- 2. IP 评分与状态机模型 (trusted / probation / blocked)
- 3. 目标站严重系统错误豁免 (500, 502, 503, 504, 520~526)
- 4. 黑名单 30 分钟 TTL 与半开自动复活 (Half-Open Revival)
- 5. 全网故障防雪崩保护熔断机制 (Global Outage Breaker)
- 6. 插件 IP 全景信誉字典查询
- """
- # 严重系统错误豁免状态码集合(目标站点自身故障,不扣除 IP 分数)
- EXEMPT_HTTP_CODES = {500, 502, 503, 504}
- CLOUDFLARE_OUTAGE_MIN = 520
- CLOUDFLARE_OUTAGE_MAX = 526
- # 黑名单封禁 TTL(分钟)
- BLOCKED_TTL_MINUTES = 30
- @classmethod
- def is_exempt_http_code(cls, http_code: int) -> bool:
- """
- 判断是否属于目标站点自身的严重系统错误(豁免扣分)
- """
- if http_code in cls.EXEMPT_HTTP_CODES:
- return True
- if cls.CLOUDFLARE_OUTAGE_MIN <= http_code <= cls.CLOUDFLARE_OUTAGE_MAX:
- return True
- return False
- @classmethod
- def is_waf_blocked(cls, http_code: int, error_type: Optional[str]) -> bool:
- """
- 判断是否命中 Cloudflare WAF 阻断 (403202 / cf-mitigated / WAF Blocked)
- """
- if not error_type and http_code == 403:
- return True
- if error_type:
- lowered = error_type.lower()
- if (
- "403202" in lowered
- or "cf-mitigated" in lowered
- or "waf blocked" in lowered
- or "waf" in lowered
- or (http_code == 403 and "mitigated" in lowered)
- ):
- return True
- return http_code == 403
- @classmethod
- async def check_global_outage_breaker(
- cls,
- db: AsyncSession,
- plugin_name: str,
- lookback_minutes: int = 3,
- min_samples: int = 5,
- outage_failure_rate_threshold: float = 0.85,
- ) -> bool:
- """
- 检查目标站全网故障保护 (Global Outage Breaker)
- 统计最近 lookback_minutes (默认3分钟) 内该插件的所有上报事件:
- 若样本数 >= min_samples 且失败率突发超过 85%,判定处于全网故障期,暂停降分与拉黑。
- """
- since_time = datetime.utcnow() - timedelta(minutes=lookback_minutes)
- stmt = (
- select(
- func.count(ProxyTelemetryLog.id).label("total_cnt"),
- func.sum(
- func.case((ProxyTelemetryLog.success.is_(False), 1), else_=0)
- ).label("fail_cnt"),
- )
- .where(
- and_(
- ProxyTelemetryLog.plugin_name == plugin_name,
- ProxyTelemetryLog.created_at >= since_time,
- )
- )
- )
- res = await db.execute(stmt)
- row = res.one_or_none()
- if not row:
- return False
- total_cnt = row.total_cnt or 0
- fail_cnt = row.fail_cnt or 0
- if total_cnt >= min_samples:
- fail_rate = fail_cnt / total_cnt
- if fail_rate > outage_failure_rate_threshold:
- logger.warning(
- f"🚨 [Global Outage Breaker] 触发全网故障保护: plugin={plugin_name}, "
- f"样本数={total_cnt}, 失败数={fail_cnt}, 失败率={fail_rate:.2%}"
- )
- return True
- return False
- @classmethod
- async def record_telemetry(
- cls,
- db: AsyncSession,
- payload: ProxyTelemetryReportRequest,
- ) -> bool:
- """
- 接收并处理客户端上报的指标数据:
- 1. 写入 proxy_telemetry_log 流水表;
- 2. 更新/初始化 proxy_reputation 信誉表;
- 3. 应用状态机、豁免规则、降级与防雪崩熔断算法。
- """
- now = datetime.utcnow()
- now_ts = payload.timestamp or int(time.time())
- # 1. 记录流水日志
- log_entry = ProxyTelemetryLog(
- client_id=payload.client_id,
- plugin_name=payload.plugin_name,
- egress_ip=payload.egress_ip,
- local_port=payload.local_port,
- node_name=payload.node_name,
- action=payload.action,
- success=payload.success,
- http_code=payload.http_code,
- error_type=payload.error_type,
- latency_ms=payload.latency_ms or 0,
- timestamp=now_ts,
- created_at=now,
- )
- db.add(log_entry)
- # 2. 检查全网目标故障保护 (Global Outage Breaker)
- is_global_outage = await cls.check_global_outage_breaker(db, payload.plugin_name)
- # 3. 查询当前 IP 的信誉记录
- stmt = select(ProxyReputation).where(
- and_(
- ProxyReputation.plugin_name == payload.plugin_name,
- ProxyReputation.egress_ip == payload.egress_ip,
- )
- )
- res = await db.execute(stmt)
- record = res.scalar_one_or_none()
- if not record:
- # 初始状态:新上报公网 IP 初始分 60 分,状态 probation
- record = ProxyReputation(
- plugin_name=payload.plugin_name,
- egress_ip=payload.egress_ip,
- score=60,
- state="probation",
- total_calls=0,
- success_calls=0,
- consecutive_successes=0,
- consecutive_fails=0,
- last_http_code=payload.http_code,
- last_error_type=payload.error_type,
- last_seen=now_ts,
- unblock_time=None,
- created_at=now,
- updated_at=now,
- )
- db.add(record)
- else:
- # 4. 黑名单租期与自动半开复活检查 (TTL & Half-Open Revival)
- if record.state == "blocked":
- if record.unblock_time and now >= record.unblock_time:
- logger.info(
- f"🔄 [Half-Open Revival] 代理节点解封复活: plugin={record.plugin_name}, ip={record.egress_ip}"
- )
- record.state = "probation"
- record.score = 50
- record.consecutive_successes = 0
- record.unblock_time = None
- # 更新调用计数与元数据
- record.total_calls += 1
- record.last_seen = now_ts
- record.last_http_code = payload.http_code
- record.last_error_type = payload.error_type
- record.updated_at = now
- # 5. 评分与状态机更新计算
- if is_global_outage:
- # 全网故障期:暂停所有 IP 的降分与拉黑计算,保持信誉字典冻结
- if payload.success:
- record.success_calls += 1
- # 记录维持不变
- logger.info(
- f"🛡️ [Breaker Active] 处于全网故障期,保持 IP 字典打分冻结: ip={record.egress_ip}, score={record.score}"
- )
- else:
- if payload.success:
- # 成功调用
- record.success_calls += 1
- record.consecutive_successes += 1
- record.consecutive_fails = 0
- # 加分规则:一次成功调用 +5 分(上限 100 分)
- record.score = min(100, record.score + 5)
- # 晋升规则:连续成功 3 次且分数 >= 75 晋升为 trusted
- if record.consecutive_successes >= 3 and record.score >= 75:
- record.state = "trusted"
- record.unblock_time = None
- elif record.state == "blocked":
- # 试探成功脱离 blocked
- record.state = "probation"
- record.unblock_time = None
- else:
- # 失败调用
- # 检查严重系统错误豁免
- if cls.is_exempt_http_code(payload.http_code):
- logger.info(
- f"🕊️ [Exempt Error] 目标系统严重故障豁免扣分: http_code={payload.http_code}, ip={record.egress_ip}"
- )
- # 豁免时不扣分,不累积失败
- else:
- record.consecutive_fails += 1
- record.consecutive_successes = 0
- # 判断是否命中 WAF 阻断
- if cls.is_waf_blocked(payload.http_code, payload.error_type):
- # 直接扣除 40 分
- record.score = max(0, record.score - 40)
- else:
- # 普通业务错误或客户端超时:扣除 2 分
- record.score = max(0, record.score - 2)
- # 状态机降级逻辑
- if record.score < 40:
- record.state = "blocked"
- record.unblock_time = now + timedelta(minutes=cls.BLOCKED_TTL_MINUTES)
- logger.warning(
- f"⛔ [Blocked] 代理节点降级拉黑: ip={record.egress_ip}, score={record.score}, "
- f"unblock_at={record.unblock_time}"
- )
- elif record.state == "trusted" and record.score < 75:
- record.state = "probation"
- await db.commit()
- return True
- @classmethod
- async def get_reputation_map(
- cls,
- db: AsyncSession,
- plugin_name: str,
- ) -> ProxyReputationMapResponse:
- """
- 获取指定插件的代理信誉全景字典
- 同时对已达 30 分钟 TTL 的 blocked IP 执行半开自动复活 (Half-Open Revival)
- """
- now = datetime.utcnow()
- stmt = select(ProxyReputation).where(ProxyReputation.plugin_name == plugin_name)
- res = await db.execute(stmt)
- records = res.scalars().all()
- reputation_map: Dict[str, ProxyReputationItemOut] = {}
- max_updated_at: Optional[datetime] = None
- has_revived = False
- for r in records:
- # 检查自动复活
- if r.state == "blocked" and r.unblock_time and now >= r.unblock_time:
- r.state = "probation"
- r.score = 50
- r.consecutive_successes = 0
- r.unblock_time = None
- r.updated_at = now
- has_revived = True
- if max_updated_at is None or (r.updated_at and r.updated_at > max_updated_at):
- max_updated_at = r.updated_at
- success_rate = (
- round(r.success_calls / r.total_calls, 2) if r.total_calls > 0 else 1.0
- )
- last_seen_ts = r.last_seen
- if not last_seen_ts:
- last_seen_ts = int(r.updated_at.timestamp()) if r.updated_at else int(time.time())
- reputation_map[r.egress_ip] = ProxyReputationItemOut(
- score=r.score,
- state=r.state,
- success_rate=success_rate,
- last_seen=last_seen_ts,
- )
- if has_revived:
- await db.commit()
- version = int(max_updated_at.timestamp()) if max_updated_at else int(time.time())
- return ProxyReputationMapResponse(
- plugin_name=plugin_name,
- version=version,
- reputation_map=reputation_map,
- )
|