redis_stream_logger.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. import time
  2. import json
  3. import queue
  4. import logging
  5. import threading
  6. from datetime import datetime, timezone
  7. from typing import Dict, Any, Optional
  8. import redis
  9. class RedisStreamLogger:
  10. """
  11. 负责将节点/角色/分组运行日志异步缓冲并通过 Redis Stream 传输到后端。
  12. 内部采用无锁内存 Queue + 后台批处理 Daemon 线程,
  13. 保证网络抖动或 Redis 延迟完全不阻塞主业务逻辑(抢号/巡检)。
  14. """
  15. _instance: Optional["RedisStreamLogger"] = None
  16. _lock = threading.Lock()
  17. def __init__(
  18. self,
  19. redis_conf: Dict[str, Any],
  20. node_id: str,
  21. role: str,
  22. maxlen: int = 2000,
  23. stream_key_prefix: str = "vs:stream:logs",
  24. ):
  25. self.node_id = node_id
  26. self.role = role
  27. self.maxlen = maxlen
  28. self.stream_key = f"{stream_key_prefix}:{node_id}:{role}"
  29. self.redis_conf = dict(redis_conf)
  30. self.redis_conf.setdefault("socket_timeout", 3.0)
  31. self.redis_conf.setdefault("socket_connect_timeout", 3.0)
  32. self.queue: queue.Queue = queue.Queue(maxsize=10000)
  33. self._stopped = False
  34. self._redis_client: Optional[redis.Redis] = None
  35. # 启动后台守护工作线程
  36. self._worker_thread = threading.Thread(
  37. target=self._flusher_loop,
  38. daemon=True,
  39. name=f"RedisStreamLogger-{role}",
  40. )
  41. self._worker_thread.start()
  42. @classmethod
  43. def initialize(
  44. cls,
  45. redis_conf: Dict[str, Any],
  46. node_id: str,
  47. role: str,
  48. maxlen: int = 2000,
  49. ) -> "RedisStreamLogger":
  50. with cls._lock:
  51. if cls._instance is None:
  52. cls._instance = cls(redis_conf, node_id, role, maxlen=maxlen)
  53. return cls._instance
  54. @classmethod
  55. def get_instance(cls) -> Optional["RedisStreamLogger"]:
  56. return cls._instance
  57. def _get_client(self) -> redis.Redis:
  58. if self._redis_client is None:
  59. self._redis_client = redis.Redis(**self.redis_conf)
  60. return self._redis_client
  61. def push(
  62. self,
  63. message: str,
  64. level: str = "INFO",
  65. group_id: str = "",
  66. timestamp: Optional[str] = None,
  67. ):
  68. """非阻塞压入待发送队列。若队列满载则丢弃旧日志,决不挂起调用方"""
  69. if self._stopped:
  70. return
  71. if not timestamp:
  72. timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
  73. payload = {
  74. "node_id": self.node_id,
  75. "role": self.role,
  76. "group_id": group_id or "",
  77. "level": level.upper(),
  78. "message": str(message),
  79. "timestamp": timestamp,
  80. }
  81. try:
  82. self.queue.put_nowait(payload)
  83. except queue.Full:
  84. try:
  85. # 丢弃一条最旧的,重新入队
  86. _ = self.queue.get_nowait()
  87. self.queue.put_nowait(payload)
  88. except Exception:
  89. pass
  90. def _flusher_loop(self):
  91. batch = []
  92. last_flush_time = time.time()
  93. while not self._stopped:
  94. try:
  95. # 尽可能凑批,最高等待 0.2 秒
  96. try:
  97. item = self.queue.get(timeout=0.2)
  98. batch.append(item)
  99. while len(batch) < 100:
  100. batch.append(self.queue.get_nowait())
  101. except queue.Empty:
  102. pass
  103. now = time.time()
  104. # 满批或超时刷新
  105. if batch and (len(batch) >= 20 or (now - last_flush_time >= 0.5)):
  106. self._flush_batch(batch)
  107. batch = []
  108. last_flush_time = now
  109. except Exception:
  110. time.sleep(0.5)
  111. def _flush_batch(self, items):
  112. if not items:
  113. return
  114. try:
  115. r = self._get_client()
  116. pipe = r.pipeline(transaction=False)
  117. for item in items:
  118. pipe.xadd(
  119. self.stream_key,
  120. item,
  121. maxlen=self.maxlen,
  122. approximate=True,
  123. )
  124. pipe.execute()
  125. except Exception:
  126. # 异常时释放连接供重试
  127. self._redis_client = None
  128. def close(self):
  129. self._stopped = True
  130. class RedisStreamLogHandler(logging.Handler):
  131. """
  132. Python logging 标准 Handler,用于透明接管 logger.info / warning / error 等日志
  133. """
  134. def __init__(self, logger_instance: Optional[RedisStreamLogger] = None):
  135. super().__init__()
  136. self._logger_instance = logger_instance
  137. def emit(self, record: logging.LogRecord):
  138. try:
  139. inst = self._logger_instance or RedisStreamLogger.get_instance()
  140. if not inst:
  141. return
  142. msg = self.format(record) if self.formatter else record.getMessage()
  143. level = record.levelname
  144. group_id = getattr(record, "group_id", "")
  145. inst.push(
  146. message=msg,
  147. level=level,
  148. group_id=group_id,
  149. )
  150. except Exception:
  151. self.handleError(record)