| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358 |
- import os
- import sys
- import time
- import json
- import queue
- import argparse
- import threading
- from typing import Dict, Any, List
- from vs_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
- from gco_wrapper import GCOWrapper
- from logger_setup import setup_app_logger
- from configure import REDIS_CFG
- from sentinel import SentinelGCO
- from booker import BuiltinBookerGCO, OrderBookerGCO
- from standalone import StandaloneGCO
- from utils.safe_redis_cli import SafeRedisClient
- from utils.cloud_config import (
- load_remote_or_cache,
- compute_config_hash,
- fetch_cloud_config,
- save_local_cache,
- )
- from utils.plugin_updater import (
- get_installed_plugins,
- get_installed_plugin_metas,
- is_newer_version,
- download_and_install_plugin,
- )
- # 升级指令队列(由 Redis PubSub 线程推送)
- upgrade_queue = queue.Queue()
- def get_gco_class(role: str, cfg: Any):
- """根据角色与配置决定具体的 GCO 实现类"""
- if role == "sentinel":
- return SentinelGCO
- if role == "standalone":
- return StandaloneGCO
- if role == "booker":
- if cfg.booker.account_source == "order":
- return OrderBookerGCO
- return BuiltinBookerGCO
- raise ValueError(f"Unknown role: {role}")
- def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
- """统一解析组配置为字典 {identifier: config_obj}"""
- items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
- cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
- result = {}
- for item in items:
- cfg = cfg_cls.from_json(item)
- result[cfg.identifier] = cfg
- return result
- def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
- """收集当前节点下所有 GroupWrapper 的聚合状态及本地已安装插件完整元数据列表"""
- groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
- installed = get_installed_plugin_metas()
- return RoleStatus(
- role=role,
- node_id=node_id,
- installed_plugins=installed,
- groups=groups_status,
- collected_at=utc_now_str(),
- )
- def reload_wrappers_for_plugin(
- plugin_name: str,
- wrappers: Dict[str, GCOWrapper],
- gco_classes: Dict[str, type],
- logger,
- ):
- """找到正在使用该插件的 Wrapper 并进行动态热重载"""
- for gid, wrapper in list(wrappers.items()):
- if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
- logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
- try:
- wrapper.stop()
- except Exception as e:
- logger.error(f"Error stopping group [{gid}] during reload: {e}")
- # 清除旧模块缓存以加载最新代码
- sys.modules.pop(plugin_name, None)
- try:
- target_cls = gco_classes[gid]
- new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
- new_wrapper.load()
- new_wrapper.start()
- wrappers[gid] = new_wrapper
- logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
- except Exception as e:
- logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
- def handle_plugin_upgrades(
- upgrade_items: List[Dict[str, Any]],
- wrappers: Dict[str, GCOWrapper],
- gco_classes: Dict[str, type],
- logger,
- ) -> bool:
- """
- 检查并执行插件升级列表
- 支持格式:
- [
- {
- "plugin_name": "tls_plugin",
- "version": "1.1.0",
- "download_url": "https://...", # 或 "code": "..."
- "filename": "tls_plugin.py"
- }
- ]
- """
- installed = get_installed_plugins()
- upgraded_any = False
- for item in upgrade_items:
- plugin_name = item.get("plugin_name")
- target_version = str(item.get("version", ""))
- if not plugin_name or not target_version:
- continue
- current_version = installed.get(plugin_name, "0.0.0")
- if is_newer_version(target_version, current_version):
- logger.info(
- f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
- )
- try:
- download_and_install_plugin(
- plugin_name=plugin_name,
- version=target_version,
- download_url=item.get("download_url"),
- code_content=item.get("code"),
- filename=item.get("filename"),
- )
- logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
- reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
- upgraded_any = True
- installed[plugin_name] = target_version
- except Exception as e:
- logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
- return upgraded_any
- def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
- """启动 Redis PubSub 后台线程,秒级监听云端下发的即时升级指令"""
- def _listen():
- import redis
- while True:
- try:
- r = redis.Redis(**redis_conf)
- pubsub = r.pubsub()
- channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
- pubsub.subscribe(*channels)
- logger.info(f"Subscribed to upgrade channels: {channels}")
- for msg in pubsub.listen():
- if msg and msg.get("type") == "message":
- try:
- raw_data = msg.get("data")
- if isinstance(raw_data, bytes):
- raw_data = raw_data.decode("utf-8")
- payload = json.loads(raw_data)
- logger.info(f"Received cloud upgrade command: {payload}")
- upgrade_queue.put(payload)
- except Exception as e:
- logger.error(f"Error parsing upgrade message: {e}")
- except Exception as e:
- logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
- time.sleep(5)
- threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
- def sync_groups(
- role: str,
- raw_conf: Any,
- wrappers: Dict[str, GCOWrapper],
- gco_classes: Dict[str, type],
- logger,
- ):
- """根据最新配置同步 GCOWrapper 实例的生命周期(停止、重启、更新、新增)"""
- new_groups = parse_group_configs(role, raw_conf)
- # 1. 停止已被移除或禁用的组
- for gid in list(wrappers.keys()):
- cfg = new_groups.get(gid)
- if not cfg or not cfg.enable:
- logger.info(f"Stopping disabled/removed group [{gid}]...")
- try:
- wrappers[gid].stop()
- except Exception as e:
- logger.error(f"Error stopping group [{gid}]: {e}")
- del wrappers[gid]
- gco_classes.pop(gid, None)
- # 2. 处理现有组更新或新增组
- for gid, cfg in new_groups.items():
- if not cfg.enable:
- continue
- target_cls = get_gco_class(role, cfg)
- if gid in wrappers:
- if gco_classes.get(gid) != target_cls:
- logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
- try:
- wrappers[gid].stop()
- except Exception as e:
- logger.error(f"Error stopping group [{gid}]: {e}")
-
- try:
- wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
- wrapper.load()
- wrapper.start()
- wrappers[gid] = wrapper
- gco_classes[gid] = target_cls
- except Exception as e:
- logger.error(f"Failed to restart group [{gid}]: {e}")
- else:
- logger.info(f"Updating configuration for group [{gid}]...")
- try:
- wrappers[gid].update_config(cfg)
- except Exception as e:
- logger.error(f"Error updating config for group [{gid}]: {e}")
- else:
- logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
- try:
- wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
- wrapper.load()
- wrapper.start()
- wrappers[gid] = wrapper
- gco_classes[gid] = target_cls
- except Exception as e:
- logger.error(f"Failed to start group [{gid}]: {e}")
- def main():
- parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
- parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
- parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
- parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
- parser.add_argument("--status-interval", type=float, default=60.0, help="RoleStatus reporting interval in seconds (default: 60s)")
- args = parser.parse_args()
- role = args.role
- use_cloud = args.config is None
- config_path = args.config or f"config/config_{role}.json"
- node_id = os.getenv("NODE_ID", "node01")
- config_key = f"{role}_config:{node_id}"
- status_key = f"vs:status:{role}:{node_id}"
- logger = setup_app_logger(role.title())
- logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
- redis_client = SafeRedisClient(REDIS_CFG, logger.info)
- # 启动云端指令实时监听(响应云端下发的升级指令)
- start_upgrade_listener(REDIS_CFG, node_id, logger)
- groups_conf, current_hash, _ = load_remote_or_cache(
- config_key=config_key,
- local_cache_path=config_path,
- use_cloud=use_cloud,
- logger=logger.info,
- )
- wrappers: Dict[str, GCOWrapper] = {}
- gco_classes: Dict[str, type] = {}
- sync_groups(role, groups_conf, wrappers, gco_classes, logger)
- logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
- # 打印本地已安装插件列表与初始 RoleStatus
- initial_status = collect_role_status(role, node_id, wrappers)
- status_dict = initial_status.to_dict()
- logger.info("📊 Initial RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
- report_ttl = max(180, int(args.status_interval * 3))
- try:
- redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
- except Exception as e:
- logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
- if use_cloud:
- logger.info(f"Cloud config active for key '{config_key}'. Polling every {args.poll_interval}s.")
- last_poll_time = time.time()
- last_status_report_time = time.time()
- try:
- while True:
- time.sleep(1)
- now = time.time()
- # 1. 响应云端下发的即时升级指令 (PubSub)
- while not upgrade_queue.empty():
- try:
- payload = upgrade_queue.get_nowait()
- plugins_items = payload.get("plugins", [])
- if isinstance(plugins_items, dict):
- plugins_items = [plugins_items]
- if plugins_items:
- if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
- # 升级成功,立即刷新并上报最新 RoleStatus
- new_status = collect_role_status(role, node_id, wrappers)
- status_dict = new_status.to_dict()
- redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
- logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
- except queue.Empty:
- break
- # 2. 定时上报包含 installed_plugins 的 RoleStatus
- if now - last_status_report_time >= args.status_interval:
- last_status_report_time = now
- try:
- current_status = collect_role_status(role, node_id, wrappers)
- status_dict = current_status.to_dict()
- redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
- logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
- except Exception as e:
- logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
- # 3. 云端分组配置定时轮询与热更新
- if use_cloud and (now - last_poll_time >= args.poll_interval):
- last_poll_time = now
- try:
- new_conf = fetch_cloud_config(config_key)
- if new_conf is not None:
- new_hash = compute_config_hash(new_conf)
- if new_hash != current_hash:
- logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
- save_local_cache(config_path, new_conf)
- sync_groups(role, new_conf, wrappers, gco_classes, logger)
- current_hash = new_hash
- logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
- except Exception as e:
- logger.warning(f"Error polling cloud config key '{config_key}': {e}")
- except KeyboardInterrupt:
- logger.info(f"Shutting down {role}...")
- for gid, wrapper in list(wrappers.items()):
- try:
- wrapper.stop()
- except Exception as e:
- logger.error(f"Error stopping group [{gid}]: {e}")
- if __name__ == "__main__":
- main()
|