redis_stream_logger.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. import time
  2. import json
  3. import queue
  4. import logging
  5. import threading
  6. from datetime import datetime, timezone
  7. from typing import Dict, Any, Optional
  8. import redis
  9. class RedisStreamLogger:
  10. """
  11. Asynchronously buffers node/role/group logs and streams them to backend via Redis Stream.
  12. Uses an in-memory lock-free Queue and background batch daemon thread,
  13. ensuring Redis latency or network jitter never blocks core operations.
  14. """
  15. _instance: Optional["RedisStreamLogger"] = None
  16. _lock = threading.Lock()
  17. def __init__(
  18. self,
  19. redis_conf: Dict[str, Any],
  20. node_id: str,
  21. role: str,
  22. maxlen: int = 2000,
  23. stream_key_prefix: str = "vs:stream:logs",
  24. ):
  25. self.node_id = node_id
  26. self.role = role
  27. self.maxlen = maxlen
  28. self.stream_key = f"{stream_key_prefix}:{node_id}:{role}"
  29. self.redis_conf = dict(redis_conf)
  30. self.redis_conf.setdefault("socket_timeout", 3.0)
  31. self.redis_conf.setdefault("socket_connect_timeout", 3.0)
  32. self.queue: queue.Queue = queue.Queue(maxsize=10000)
  33. self._stopped = False
  34. self._redis_client: Optional[redis.Redis] = None
  35. # Start background daemon worker thread
  36. self._worker_thread = threading.Thread(
  37. target=self._flusher_loop,
  38. daemon=True,
  39. name=f"RedisStreamLogger-{role}",
  40. )
  41. self._worker_thread.start()
  42. @classmethod
  43. def initialize(
  44. cls,
  45. redis_conf: Dict[str, Any],
  46. node_id: str,
  47. role: str,
  48. maxlen: int = 2000,
  49. ) -> "RedisStreamLogger":
  50. with cls._lock:
  51. if cls._instance is None:
  52. cls._instance = cls(redis_conf, node_id, role, maxlen=maxlen)
  53. return cls._instance
  54. @classmethod
  55. def get_instance(cls) -> Optional["RedisStreamLogger"]:
  56. return cls._instance
  57. def _get_client(self) -> redis.Redis:
  58. if self._redis_client is None:
  59. self._redis_client = redis.Redis(**self.redis_conf)
  60. return self._redis_client
  61. def push(
  62. self,
  63. message: str,
  64. level: str = "INFO",
  65. group_id: str = "",
  66. timestamp: Optional[str] = None,
  67. ):
  68. """Non-blocking enqueue. Drops oldest log entry if queue is full, never hanging caller."""
  69. if self._stopped:
  70. return
  71. if not timestamp:
  72. timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
  73. payload = {
  74. "node_id": self.node_id,
  75. "role": self.role,
  76. "group_id": group_id or "",
  77. "level": level.upper(),
  78. "message": str(message),
  79. "timestamp": timestamp,
  80. }
  81. try:
  82. self.queue.put_nowait(payload)
  83. except queue.Full:
  84. try:
  85. # Drop oldest item and re-enqueue
  86. _ = self.queue.get_nowait()
  87. self.queue.put_nowait(payload)
  88. except Exception:
  89. pass
  90. def _flusher_loop(self):
  91. batch = []
  92. last_flush_time = time.time()
  93. while not self._stopped:
  94. try:
  95. # Batch items up to 0.2s
  96. try:
  97. item = self.queue.get(timeout=0.2)
  98. batch.append(item)
  99. while len(batch) < 100:
  100. batch.append(self.queue.get_nowait())
  101. except queue.Empty:
  102. pass
  103. now = time.time()
  104. # Flush if full batch or timeout
  105. if batch and (len(batch) >= 20 or (now - last_flush_time >= 0.5)):
  106. self._flush_batch(batch)
  107. batch = []
  108. last_flush_time = now
  109. except Exception:
  110. time.sleep(0.5)
  111. def _flush_batch(self, items):
  112. if not items:
  113. return
  114. try:
  115. r = self._get_client()
  116. pipe = r.pipeline(transaction=False)
  117. for item in items:
  118. pipe.xadd(
  119. self.stream_key,
  120. item,
  121. maxlen=self.maxlen,
  122. approximate=True,
  123. )
  124. pipe.execute()
  125. except Exception:
  126. # Release client connection on error to allow reconnection
  127. self._redis_client = None
  128. def close(self):
  129. self._stopped = True
  130. class RedisStreamLogHandler(logging.Handler):
  131. """
  132. Standard Python logging Handler to transparently forward log records to Redis Stream.
  133. """
  134. def __init__(self, logger_instance: Optional[RedisStreamLogger] = None):
  135. super().__init__()
  136. self._logger_instance = logger_instance
  137. def emit(self, record: logging.LogRecord):
  138. try:
  139. inst = self._logger_instance or RedisStreamLogger.get_instance()
  140. if not inst:
  141. return
  142. msg = self.format(record) if self.formatter else record.getMessage()
  143. level = record.levelname
  144. group_id = getattr(record, "group_id", "")
  145. inst.push(
  146. message=msg,
  147. level=level,
  148. group_id=group_id,
  149. )
  150. except Exception:
  151. self.handleError(record)