| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189 |
- import time
- import json
- import queue
- import logging
- import threading
- from datetime import datetime, timezone
- from typing import Dict, Any, Optional
- import redis
- class RedisStreamLogger:
- """
- Asynchronously buffers node/role/group logs and streams them to backend via Redis Stream.
- Uses an in-memory lock-free Queue and background batch daemon thread,
- ensuring Redis latency or network jitter never blocks core operations.
- """
- _instance: Optional["RedisStreamLogger"] = None
- _lock = threading.Lock()
- def __init__(
- self,
- redis_conf: Dict[str, Any],
- node_id: str,
- role: str,
- maxlen: int = 2000,
- stream_key_prefix: str = "vs:stream:logs",
- ):
- self.node_id = node_id
- self.role = role
- self.maxlen = maxlen
- self.stream_key = f"{stream_key_prefix}:{node_id}:{role}"
- self.redis_conf = dict(redis_conf)
- self.redis_conf.setdefault("socket_timeout", 3.0)
- self.redis_conf.setdefault("socket_connect_timeout", 3.0)
-
- self.queue: queue.Queue = queue.Queue(maxsize=10000)
- self._stopped = False
- self._redis_client: Optional[redis.Redis] = None
-
- # Start background daemon worker thread
- self._worker_thread = threading.Thread(
- target=self._flusher_loop,
- daemon=True,
- name=f"RedisStreamLogger-{role}",
- )
- self._worker_thread.start()
- @classmethod
- def initialize(
- cls,
- redis_conf: Dict[str, Any],
- node_id: str,
- role: str,
- maxlen: int = 2000,
- ) -> "RedisStreamLogger":
- with cls._lock:
- if cls._instance is None:
- cls._instance = cls(redis_conf, node_id, role, maxlen=maxlen)
- return cls._instance
- @classmethod
- def get_instance(cls) -> Optional["RedisStreamLogger"]:
- return cls._instance
- def _get_client(self) -> redis.Redis:
- if self._stopped:
- raise RuntimeError('Stream logger is closed')
- if self._redis_client is None:
- self._redis_client = redis.Redis(**self.redis_conf)
- return self._redis_client
- def push(
- self,
- message: str,
- level: str = "INFO",
- group_id: str = "",
- timestamp: Optional[str] = None,
- ):
- """Non-blocking enqueue. Drops oldest log entry if queue is full, never hanging caller."""
- if self._stopped:
- return
- if not timestamp:
- timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
- payload = {
- "node_id": self.node_id,
- "role": self.role,
- "group_id": group_id or "",
- "level": level.upper(),
- "message": str(message),
- "timestamp": timestamp,
- }
- try:
- self.queue.put_nowait(payload)
- except queue.Full:
- try:
- # Drop oldest item and re-enqueue
- _ = self.queue.get_nowait()
- self.queue.put_nowait(payload)
- except Exception:
- pass
- def _flusher_loop(self):
- batch = []
- last_flush_time = time.time()
- while not self._stopped:
- try:
- # Batch items up to 0.2s
- try:
- item = self.queue.get(timeout=0.2)
- batch.append(item)
- while len(batch) < 100:
- batch.append(self.queue.get_nowait())
- except queue.Empty:
- pass
- now = time.time()
- # Flush if full batch or timeout
- if batch and (len(batch) >= 20 or (now - last_flush_time >= 0.5)):
- self._flush_batch(batch)
- batch = []
- last_flush_time = now
- except Exception:
- time.sleep(0.5)
- def _flush_batch(self, items):
- if not items:
- return
- try:
- r = self._get_client()
- pipe = r.pipeline(transaction=False)
- for item in items:
- pipe.xadd(
- self.stream_key,
- item,
- maxlen=self.maxlen,
- approximate=True,
- )
- pipe.execute()
- except Exception:
- # Release client connection on error to allow reconnection
- if self._redis_client is not None:
- self._redis_client.close()
- self._redis_client = None
- def close(self):
- self._stopped = True
- if self._worker_thread is not threading.current_thread():
- self._worker_thread.join(2)
- client = self._redis_client
- self._redis_client = None
- if client is not None:
- client.close()
- client.connection_pool.disconnect()
- if self._worker_thread is not threading.current_thread():
- self._worker_thread.join(1)
- with type(self)._lock:
- if type(self)._instance is self:
- type(self)._instance = None
- class RedisStreamLogHandler(logging.Handler):
- """
- Standard Python logging Handler to transparently forward log records to Redis Stream.
- """
- def __init__(self, logger_instance: Optional[RedisStreamLogger] = None):
- super().__init__()
- self._logger_instance = logger_instance
- def emit(self, record: logging.LogRecord):
- try:
- inst = self._logger_instance or RedisStreamLogger.get_instance()
- if not inst:
- return
- msg = self.format(record) if self.formatter else record.getMessage()
- level = record.levelname
- group_id = getattr(record, "group_id", "")
- inst.push(
- message=msg,
- level=level,
- group_id=group_id,
- )
- except Exception:
- self.handleError(record)
|