|
@@ -0,0 +1,173 @@
|
|
|
|
|
+import time
|
|
|
|
|
+import json
|
|
|
|
|
+import queue
|
|
|
|
|
+import logging
|
|
|
|
|
+import threading
|
|
|
|
|
+from datetime import datetime, timezone
|
|
|
|
|
+from typing import Dict, Any, Optional
|
|
|
|
|
+import redis
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class RedisStreamLogger:
|
|
|
|
|
+ """
|
|
|
|
|
+ 负责将节点/角色/分组运行日志异步缓冲并通过 Redis Stream 传输到后端。
|
|
|
|
|
+ 内部采用无锁内存 Queue + 后台批处理 Daemon 线程,
|
|
|
|
|
+ 保证网络抖动或 Redis 延迟完全不阻塞主业务逻辑(抢号/巡检)。
|
|
|
|
|
+ """
|
|
|
|
|
+ _instance: Optional["RedisStreamLogger"] = None
|
|
|
|
|
+ _lock = threading.Lock()
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ redis_conf: Dict[str, Any],
|
|
|
|
|
+ node_id: str,
|
|
|
|
|
+ role: str,
|
|
|
|
|
+ maxlen: int = 2000,
|
|
|
|
|
+ stream_key_prefix: str = "vs:stream:logs",
|
|
|
|
|
+ ):
|
|
|
|
|
+ self.node_id = node_id
|
|
|
|
|
+ self.role = role
|
|
|
|
|
+ self.maxlen = maxlen
|
|
|
|
|
+ self.stream_key = f"{stream_key_prefix}:{node_id}:{role}"
|
|
|
|
|
+ self.redis_conf = dict(redis_conf)
|
|
|
|
|
+ self.redis_conf.setdefault("socket_timeout", 3.0)
|
|
|
|
|
+ self.redis_conf.setdefault("socket_connect_timeout", 3.0)
|
|
|
|
|
+
|
|
|
|
|
+ self.queue: queue.Queue = queue.Queue(maxsize=10000)
|
|
|
|
|
+ self._stopped = False
|
|
|
|
|
+ self._redis_client: Optional[redis.Redis] = None
|
|
|
|
|
+
|
|
|
|
|
+ # 启动后台守护工作线程
|
|
|
|
|
+ self._worker_thread = threading.Thread(
|
|
|
|
|
+ target=self._flusher_loop,
|
|
|
|
|
+ daemon=True,
|
|
|
|
|
+ name=f"RedisStreamLogger-{role}",
|
|
|
|
|
+ )
|
|
|
|
|
+ self._worker_thread.start()
|
|
|
|
|
+
|
|
|
|
|
+ @classmethod
|
|
|
|
|
+ def initialize(
|
|
|
|
|
+ cls,
|
|
|
|
|
+ redis_conf: Dict[str, Any],
|
|
|
|
|
+ node_id: str,
|
|
|
|
|
+ role: str,
|
|
|
|
|
+ maxlen: int = 2000,
|
|
|
|
|
+ ) -> "RedisStreamLogger":
|
|
|
|
|
+ with cls._lock:
|
|
|
|
|
+ if cls._instance is None:
|
|
|
|
|
+ cls._instance = cls(redis_conf, node_id, role, maxlen=maxlen)
|
|
|
|
|
+ return cls._instance
|
|
|
|
|
+
|
|
|
|
|
+ @classmethod
|
|
|
|
|
+ def get_instance(cls) -> Optional["RedisStreamLogger"]:
|
|
|
|
|
+ return cls._instance
|
|
|
|
|
+
|
|
|
|
|
+ def _get_client(self) -> redis.Redis:
|
|
|
|
|
+ if self._redis_client is None:
|
|
|
|
|
+ self._redis_client = redis.Redis(**self.redis_conf)
|
|
|
|
|
+ return self._redis_client
|
|
|
|
|
+
|
|
|
|
|
+ def push(
|
|
|
|
|
+ self,
|
|
|
|
|
+ message: str,
|
|
|
|
|
+ level: str = "INFO",
|
|
|
|
|
+ group_id: str = "",
|
|
|
|
|
+ timestamp: Optional[str] = None,
|
|
|
|
|
+ ):
|
|
|
|
|
+ """非阻塞压入待发送队列。若队列满载则丢弃旧日志,决不挂起调用方"""
|
|
|
|
|
+ if self._stopped:
|
|
|
|
|
+ return
|
|
|
|
|
+ if not timestamp:
|
|
|
|
|
+ timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
+
|
|
|
|
|
+ payload = {
|
|
|
|
|
+ "node_id": self.node_id,
|
|
|
|
|
+ "role": self.role,
|
|
|
|
|
+ "group_id": group_id or "",
|
|
|
|
|
+ "level": level.upper(),
|
|
|
|
|
+ "message": str(message),
|
|
|
|
|
+ "timestamp": timestamp,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ try:
|
|
|
|
|
+ self.queue.put_nowait(payload)
|
|
|
|
|
+ except queue.Full:
|
|
|
|
|
+ try:
|
|
|
|
|
+ # 丢弃一条最旧的,重新入队
|
|
|
|
|
+ _ = self.queue.get_nowait()
|
|
|
|
|
+ self.queue.put_nowait(payload)
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ def _flusher_loop(self):
|
|
|
|
|
+ batch = []
|
|
|
|
|
+ last_flush_time = time.time()
|
|
|
|
|
+
|
|
|
|
|
+ while not self._stopped:
|
|
|
|
|
+ try:
|
|
|
|
|
+ # 尽可能凑批,最高等待 0.2 秒
|
|
|
|
|
+ try:
|
|
|
|
|
+ item = self.queue.get(timeout=0.2)
|
|
|
|
|
+ batch.append(item)
|
|
|
|
|
+ while len(batch) < 100:
|
|
|
|
|
+ batch.append(self.queue.get_nowait())
|
|
|
|
|
+ except queue.Empty:
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ now = time.time()
|
|
|
|
|
+ # 满批或超时刷新
|
|
|
|
|
+ if batch and (len(batch) >= 20 or (now - last_flush_time >= 0.5)):
|
|
|
|
|
+ self._flush_batch(batch)
|
|
|
|
|
+ batch = []
|
|
|
|
|
+ last_flush_time = now
|
|
|
|
|
+
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ time.sleep(0.5)
|
|
|
|
|
+
|
|
|
|
|
+ def _flush_batch(self, items):
|
|
|
|
|
+ if not items:
|
|
|
|
|
+ return
|
|
|
|
|
+ try:
|
|
|
|
|
+ r = self._get_client()
|
|
|
|
|
+ pipe = r.pipeline(transaction=False)
|
|
|
|
|
+ for item in items:
|
|
|
|
|
+ pipe.xadd(
|
|
|
|
|
+ self.stream_key,
|
|
|
|
|
+ item,
|
|
|
|
|
+ maxlen=self.maxlen,
|
|
|
|
|
+ approximate=True,
|
|
|
|
|
+ )
|
|
|
|
|
+ pipe.execute()
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ # 异常时释放连接供重试
|
|
|
|
|
+ self._redis_client = None
|
|
|
|
|
+
|
|
|
|
|
+ def close(self):
|
|
|
|
|
+ self._stopped = True
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class RedisStreamLogHandler(logging.Handler):
|
|
|
|
|
+ """
|
|
|
|
|
+ Python logging 标准 Handler,用于透明接管 logger.info / warning / error 等日志
|
|
|
|
|
+ """
|
|
|
|
|
+ def __init__(self, logger_instance: Optional[RedisStreamLogger] = None):
|
|
|
|
|
+ super().__init__()
|
|
|
|
|
+ self._logger_instance = logger_instance
|
|
|
|
|
+
|
|
|
|
|
+ def emit(self, record: logging.LogRecord):
|
|
|
|
|
+ try:
|
|
|
|
|
+ inst = self._logger_instance or RedisStreamLogger.get_instance()
|
|
|
|
|
+ if not inst:
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+ msg = self.format(record) if self.formatter else record.getMessage()
|
|
|
|
|
+ level = record.levelname
|
|
|
|
|
+ group_id = getattr(record, "group_id", "")
|
|
|
|
|
+
|
|
|
|
|
+ inst.push(
|
|
|
|
|
+ message=msg,
|
|
|
|
|
+ level=level,
|
|
|
|
|
+ group_id=group_id,
|
|
|
|
|
+ )
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ self.handleError(record)
|