redis_stream_logger.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189
  1. import time
  2. import json
  3. import queue
  4. import logging
  5. import threading
  6. from datetime import datetime, timezone
  7. from typing import Dict, Any, Optional
  8. import redis
  9. class RedisStreamLogger:
  10. """
  11. Asynchronously buffers node/role/group logs and streams them to backend via Redis Stream.
  12. Uses an in-memory lock-free Queue and background batch daemon thread,
  13. ensuring Redis latency or network jitter never blocks core operations.
  14. """
  15. _instance: Optional["RedisStreamLogger"] = None
  16. _lock = threading.Lock()
  17. def __init__(
  18. self,
  19. redis_conf: Dict[str, Any],
  20. node_id: str,
  21. role: str,
  22. maxlen: int = 2000,
  23. stream_key_prefix: str = "vs:stream:logs",
  24. ):
  25. self.node_id = node_id
  26. self.role = role
  27. self.maxlen = maxlen
  28. self.stream_key = f"{stream_key_prefix}:{node_id}:{role}"
  29. self.redis_conf = dict(redis_conf)
  30. self.redis_conf.setdefault("socket_timeout", 3.0)
  31. self.redis_conf.setdefault("socket_connect_timeout", 3.0)
  32. self.queue: queue.Queue = queue.Queue(maxsize=10000)
  33. self._stopped = False
  34. self._redis_client: Optional[redis.Redis] = None
  35. # Start background daemon worker thread
  36. self._worker_thread = threading.Thread(
  37. target=self._flusher_loop,
  38. daemon=True,
  39. name=f"RedisStreamLogger-{role}",
  40. )
  41. self._worker_thread.start()
  42. @classmethod
  43. def initialize(
  44. cls,
  45. redis_conf: Dict[str, Any],
  46. node_id: str,
  47. role: str,
  48. maxlen: int = 2000,
  49. ) -> "RedisStreamLogger":
  50. with cls._lock:
  51. if cls._instance is None:
  52. cls._instance = cls(redis_conf, node_id, role, maxlen=maxlen)
  53. return cls._instance
  54. @classmethod
  55. def get_instance(cls) -> Optional["RedisStreamLogger"]:
  56. return cls._instance
  57. def _get_client(self) -> redis.Redis:
  58. if self._stopped:
  59. raise RuntimeError('Stream logger is closed')
  60. if self._redis_client is None:
  61. self._redis_client = redis.Redis(**self.redis_conf)
  62. return self._redis_client
  63. def push(
  64. self,
  65. message: str,
  66. level: str = "INFO",
  67. group_id: str = "",
  68. timestamp: Optional[str] = None,
  69. ):
  70. """Non-blocking enqueue. Drops oldest log entry if queue is full, never hanging caller."""
  71. if self._stopped:
  72. return
  73. if not timestamp:
  74. timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
  75. payload = {
  76. "node_id": self.node_id,
  77. "role": self.role,
  78. "group_id": group_id or "",
  79. "level": level.upper(),
  80. "message": str(message),
  81. "timestamp": timestamp,
  82. }
  83. try:
  84. self.queue.put_nowait(payload)
  85. except queue.Full:
  86. try:
  87. # Drop oldest item and re-enqueue
  88. _ = self.queue.get_nowait()
  89. self.queue.put_nowait(payload)
  90. except Exception:
  91. pass
  92. def _flusher_loop(self):
  93. batch = []
  94. last_flush_time = time.time()
  95. while not self._stopped:
  96. try:
  97. # Batch items up to 0.2s
  98. try:
  99. item = self.queue.get(timeout=0.2)
  100. batch.append(item)
  101. while len(batch) < 100:
  102. batch.append(self.queue.get_nowait())
  103. except queue.Empty:
  104. pass
  105. now = time.time()
  106. # Flush if full batch or timeout
  107. if batch and (len(batch) >= 20 or (now - last_flush_time >= 0.5)):
  108. self._flush_batch(batch)
  109. batch = []
  110. last_flush_time = now
  111. except Exception:
  112. time.sleep(0.5)
  113. def _flush_batch(self, items):
  114. if not items:
  115. return
  116. try:
  117. r = self._get_client()
  118. pipe = r.pipeline(transaction=False)
  119. for item in items:
  120. pipe.xadd(
  121. self.stream_key,
  122. item,
  123. maxlen=self.maxlen,
  124. approximate=True,
  125. )
  126. pipe.execute()
  127. except Exception:
  128. # Release client connection on error to allow reconnection
  129. if self._redis_client is not None:
  130. self._redis_client.close()
  131. self._redis_client = None
  132. def close(self):
  133. self._stopped = True
  134. if self._worker_thread is not threading.current_thread():
  135. self._worker_thread.join(2)
  136. client = self._redis_client
  137. self._redis_client = None
  138. if client is not None:
  139. client.close()
  140. client.connection_pool.disconnect()
  141. if self._worker_thread is not threading.current_thread():
  142. self._worker_thread.join(1)
  143. with type(self)._lock:
  144. if type(self)._instance is self:
  145. type(self)._instance = None
  146. class RedisStreamLogHandler(logging.Handler):
  147. """
  148. Standard Python logging Handler to transparently forward log records to Redis Stream.
  149. """
  150. def __init__(self, logger_instance: Optional[RedisStreamLogger] = None):
  151. super().__init__()
  152. self._logger_instance = logger_instance
  153. def emit(self, record: logging.LogRecord):
  154. try:
  155. inst = self._logger_instance or RedisStreamLogger.get_instance()
  156. if not inst:
  157. return
  158. msg = self.format(record) if self.formatter else record.getMessage()
  159. level = record.levelname
  160. group_id = getattr(record, "group_id", "")
  161. inst.push(
  162. message=msg,
  163. level=level,
  164. group_id=group_id,
  165. )
  166. except Exception:
  167. self.handleError(record)