import time import json import queue import logging import threading from datetime import datetime, timezone from typing import Dict, Any, Optional import redis class RedisStreamLogger: """ Asynchronously buffers node/role/group logs and streams them to backend via Redis Stream. Uses an in-memory lock-free Queue and background batch daemon thread, ensuring Redis latency or network jitter never blocks core operations. """ _instance: Optional["RedisStreamLogger"] = None _lock = threading.Lock() def __init__( self, redis_conf: Dict[str, Any], node_id: str, role: str, maxlen: int = 2000, stream_key_prefix: str = "vs:stream:logs", ): self.node_id = node_id self.role = role self.maxlen = maxlen self.stream_key = f"{stream_key_prefix}:{node_id}:{role}" self.redis_conf = dict(redis_conf) self.redis_conf.setdefault("socket_timeout", 3.0) self.redis_conf.setdefault("socket_connect_timeout", 3.0) self.queue: queue.Queue = queue.Queue(maxsize=10000) self._stopped = False self._redis_client: Optional[redis.Redis] = None # Start background daemon worker thread self._worker_thread = threading.Thread( target=self._flusher_loop, daemon=True, name=f"RedisStreamLogger-{role}", ) self._worker_thread.start() @classmethod def initialize( cls, redis_conf: Dict[str, Any], node_id: str, role: str, maxlen: int = 2000, ) -> "RedisStreamLogger": with cls._lock: if cls._instance is None: cls._instance = cls(redis_conf, node_id, role, maxlen=maxlen) return cls._instance @classmethod def get_instance(cls) -> Optional["RedisStreamLogger"]: return cls._instance def _get_client(self) -> redis.Redis: if self._stopped: raise RuntimeError('Stream logger is closed') if self._redis_client is None: self._redis_client = redis.Redis(**self.redis_conf) return self._redis_client def push( self, message: str, level: str = "INFO", group_id: str = "", timestamp: Optional[str] = None, ): """Non-blocking enqueue. Drops oldest log entry if queue is full, never hanging caller.""" if self._stopped: return if not timestamp: timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") payload = { "node_id": self.node_id, "role": self.role, "group_id": group_id or "", "level": level.upper(), "message": str(message), "timestamp": timestamp, } try: self.queue.put_nowait(payload) except queue.Full: try: # Drop oldest item and re-enqueue _ = self.queue.get_nowait() self.queue.put_nowait(payload) except Exception: pass def _flusher_loop(self): batch = [] last_flush_time = time.time() while not self._stopped: try: # Batch items up to 0.2s try: item = self.queue.get(timeout=0.2) batch.append(item) while len(batch) < 100: batch.append(self.queue.get_nowait()) except queue.Empty: pass now = time.time() # Flush if full batch or timeout if batch and (len(batch) >= 20 or (now - last_flush_time >= 0.5)): self._flush_batch(batch) batch = [] last_flush_time = now except Exception: time.sleep(0.5) def _flush_batch(self, items): if not items: return try: r = self._get_client() pipe = r.pipeline(transaction=False) for item in items: pipe.xadd( self.stream_key, item, maxlen=self.maxlen, approximate=True, ) pipe.execute() except Exception: # Release client connection on error to allow reconnection if self._redis_client is not None: self._redis_client.close() self._redis_client = None def close(self): self._stopped = True if self._worker_thread is not threading.current_thread(): self._worker_thread.join(2) client = self._redis_client self._redis_client = None if client is not None: client.close() client.connection_pool.disconnect() if self._worker_thread is not threading.current_thread(): self._worker_thread.join(1) with type(self)._lock: if type(self)._instance is self: type(self)._instance = None class RedisStreamLogHandler(logging.Handler): """ Standard Python logging Handler to transparently forward log records to Redis Stream. """ def __init__(self, logger_instance: Optional[RedisStreamLogger] = None): super().__init__() self._logger_instance = logger_instance def emit(self, record: logging.LogRecord): try: inst = self._logger_instance or RedisStreamLogger.get_instance() if not inst: return msg = self.format(record) if self.formatter else record.getMessage() level = record.levelname group_id = getattr(record, "group_id", "") inst.push( message=msg, level=level, group_id=group_id, ) except Exception: self.handleError(record)