|
|
@@ -0,0 +1,358 @@
|
|
|
+import os
|
|
|
+import sys
|
|
|
+import time
|
|
|
+import json
|
|
|
+import queue
|
|
|
+import argparse
|
|
|
+import threading
|
|
|
+from typing import Dict, Any, List
|
|
|
+
|
|
|
+from vs_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
|
|
|
+from gco_wrapper import GCOWrapper
|
|
|
+from logger_setup import setup_app_logger
|
|
|
+from configure import REDIS_CFG
|
|
|
+from sentinel import SentinelGCO
|
|
|
+from booker import BuiltinBookerGCO, OrderBookerGCO
|
|
|
+from standalone import StandaloneGCO
|
|
|
+from utils.safe_redis_cli import SafeRedisClient
|
|
|
+from utils.cloud_config import (
|
|
|
+ load_remote_or_cache,
|
|
|
+ compute_config_hash,
|
|
|
+ fetch_cloud_config,
|
|
|
+ save_local_cache,
|
|
|
+)
|
|
|
+from utils.plugin_updater import (
|
|
|
+ get_installed_plugins,
|
|
|
+ get_installed_plugin_metas,
|
|
|
+ is_newer_version,
|
|
|
+ download_and_install_plugin,
|
|
|
+)
|
|
|
+
|
|
|
+# 升级指令队列(由 Redis PubSub 线程推送)
|
|
|
+upgrade_queue = queue.Queue()
|
|
|
+
|
|
|
+
|
|
|
+def get_gco_class(role: str, cfg: Any):
|
|
|
+ """根据角色与配置决定具体的 GCO 实现类"""
|
|
|
+ if role == "sentinel":
|
|
|
+ return SentinelGCO
|
|
|
+ if role == "standalone":
|
|
|
+ return StandaloneGCO
|
|
|
+ if role == "booker":
|
|
|
+ if cfg.booker.account_source == "order":
|
|
|
+ return OrderBookerGCO
|
|
|
+ return BuiltinBookerGCO
|
|
|
+ raise ValueError(f"Unknown role: {role}")
|
|
|
+
|
|
|
+
|
|
|
+def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
|
|
|
+ """统一解析组配置为字典 {identifier: config_obj}"""
|
|
|
+ items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
|
|
|
+ cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
|
|
|
+
|
|
|
+ result = {}
|
|
|
+ for item in items:
|
|
|
+ cfg = cfg_cls.from_json(item)
|
|
|
+ result[cfg.identifier] = cfg
|
|
|
+ return result
|
|
|
+
|
|
|
+
|
|
|
+def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
|
|
|
+ """收集当前节点下所有 GroupWrapper 的聚合状态及本地已安装插件完整元数据列表"""
|
|
|
+ groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
|
|
|
+ installed = get_installed_plugin_metas()
|
|
|
+
|
|
|
+ return RoleStatus(
|
|
|
+ role=role,
|
|
|
+ node_id=node_id,
|
|
|
+ installed_plugins=installed,
|
|
|
+ groups=groups_status,
|
|
|
+ collected_at=utc_now_str(),
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def reload_wrappers_for_plugin(
|
|
|
+ plugin_name: str,
|
|
|
+ wrappers: Dict[str, GCOWrapper],
|
|
|
+ gco_classes: Dict[str, type],
|
|
|
+ logger,
|
|
|
+):
|
|
|
+ """找到正在使用该插件的 Wrapper 并进行动态热重载"""
|
|
|
+ for gid, wrapper in list(wrappers.items()):
|
|
|
+ if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
|
|
|
+ logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
|
|
|
+ try:
|
|
|
+ wrapper.stop()
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Error stopping group [{gid}] during reload: {e}")
|
|
|
+
|
|
|
+ # 清除旧模块缓存以加载最新代码
|
|
|
+ sys.modules.pop(plugin_name, None)
|
|
|
+
|
|
|
+ try:
|
|
|
+ target_cls = gco_classes[gid]
|
|
|
+ new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
|
|
|
+ new_wrapper.load()
|
|
|
+ new_wrapper.start()
|
|
|
+ wrappers[gid] = new_wrapper
|
|
|
+ logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
|
|
|
+
|
|
|
+
|
|
|
+def handle_plugin_upgrades(
|
|
|
+ upgrade_items: List[Dict[str, Any]],
|
|
|
+ wrappers: Dict[str, GCOWrapper],
|
|
|
+ gco_classes: Dict[str, type],
|
|
|
+ logger,
|
|
|
+) -> bool:
|
|
|
+ """
|
|
|
+ 检查并执行插件升级列表
|
|
|
+ 支持格式:
|
|
|
+ [
|
|
|
+ {
|
|
|
+ "plugin_name": "tls_plugin",
|
|
|
+ "version": "1.1.0",
|
|
|
+ "download_url": "https://...", # 或 "code": "..."
|
|
|
+ "filename": "tls_plugin.py"
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ """
|
|
|
+ installed = get_installed_plugins()
|
|
|
+ upgraded_any = False
|
|
|
+
|
|
|
+ for item in upgrade_items:
|
|
|
+ plugin_name = item.get("plugin_name")
|
|
|
+ target_version = str(item.get("version", ""))
|
|
|
+ if not plugin_name or not target_version:
|
|
|
+ continue
|
|
|
+
|
|
|
+ current_version = installed.get(plugin_name, "0.0.0")
|
|
|
+ if is_newer_version(target_version, current_version):
|
|
|
+ logger.info(
|
|
|
+ f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
|
|
|
+ )
|
|
|
+ try:
|
|
|
+ download_and_install_plugin(
|
|
|
+ plugin_name=plugin_name,
|
|
|
+ version=target_version,
|
|
|
+ download_url=item.get("download_url"),
|
|
|
+ code_content=item.get("code"),
|
|
|
+ filename=item.get("filename"),
|
|
|
+ )
|
|
|
+ logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
|
|
|
+ reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
|
|
|
+ upgraded_any = True
|
|
|
+ installed[plugin_name] = target_version
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
|
|
|
+
|
|
|
+ return upgraded_any
|
|
|
+
|
|
|
+
|
|
|
+def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
|
|
|
+ """启动 Redis PubSub 后台线程,秒级监听云端下发的即时升级指令"""
|
|
|
+ def _listen():
|
|
|
+ import redis
|
|
|
+ while True:
|
|
|
+ try:
|
|
|
+ r = redis.Redis(**redis_conf)
|
|
|
+ pubsub = r.pubsub()
|
|
|
+ channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
|
|
|
+ pubsub.subscribe(*channels)
|
|
|
+ logger.info(f"Subscribed to upgrade channels: {channels}")
|
|
|
+
|
|
|
+ for msg in pubsub.listen():
|
|
|
+ if msg and msg.get("type") == "message":
|
|
|
+ try:
|
|
|
+ raw_data = msg.get("data")
|
|
|
+ if isinstance(raw_data, bytes):
|
|
|
+ raw_data = raw_data.decode("utf-8")
|
|
|
+ payload = json.loads(raw_data)
|
|
|
+ logger.info(f"Received cloud upgrade command: {payload}")
|
|
|
+ upgrade_queue.put(payload)
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Error parsing upgrade message: {e}")
|
|
|
+ except Exception as e:
|
|
|
+ logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
|
|
|
+ time.sleep(5)
|
|
|
+
|
|
|
+ threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
|
|
|
+
|
|
|
+
|
|
|
+def sync_groups(
|
|
|
+ role: str,
|
|
|
+ raw_conf: Any,
|
|
|
+ wrappers: Dict[str, GCOWrapper],
|
|
|
+ gco_classes: Dict[str, type],
|
|
|
+ logger,
|
|
|
+):
|
|
|
+ """根据最新配置同步 GCOWrapper 实例的生命周期(停止、重启、更新、新增)"""
|
|
|
+ new_groups = parse_group_configs(role, raw_conf)
|
|
|
+
|
|
|
+ # 1. 停止已被移除或禁用的组
|
|
|
+ for gid in list(wrappers.keys()):
|
|
|
+ cfg = new_groups.get(gid)
|
|
|
+ if not cfg or not cfg.enable:
|
|
|
+ logger.info(f"Stopping disabled/removed group [{gid}]...")
|
|
|
+ try:
|
|
|
+ wrappers[gid].stop()
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Error stopping group [{gid}]: {e}")
|
|
|
+ del wrappers[gid]
|
|
|
+ gco_classes.pop(gid, None)
|
|
|
+
|
|
|
+ # 2. 处理现有组更新或新增组
|
|
|
+ for gid, cfg in new_groups.items():
|
|
|
+ if not cfg.enable:
|
|
|
+ continue
|
|
|
+
|
|
|
+ target_cls = get_gco_class(role, cfg)
|
|
|
+
|
|
|
+ if gid in wrappers:
|
|
|
+ if gco_classes.get(gid) != target_cls:
|
|
|
+ logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
|
|
|
+ try:
|
|
|
+ wrappers[gid].stop()
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Error stopping group [{gid}]: {e}")
|
|
|
+
|
|
|
+ try:
|
|
|
+ wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
|
|
|
+ wrapper.load()
|
|
|
+ wrapper.start()
|
|
|
+ wrappers[gid] = wrapper
|
|
|
+ gco_classes[gid] = target_cls
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Failed to restart group [{gid}]: {e}")
|
|
|
+ else:
|
|
|
+ logger.info(f"Updating configuration for group [{gid}]...")
|
|
|
+ try:
|
|
|
+ wrappers[gid].update_config(cfg)
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Error updating config for group [{gid}]: {e}")
|
|
|
+ else:
|
|
|
+ logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
|
|
|
+ try:
|
|
|
+ wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
|
|
|
+ wrapper.load()
|
|
|
+ wrapper.start()
|
|
|
+ wrappers[gid] = wrapper
|
|
|
+ gco_classes[gid] = target_cls
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Failed to start group [{gid}]: {e}")
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+ parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
|
|
|
+ parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
|
|
|
+ parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
|
|
|
+ parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
|
|
|
+ parser.add_argument("--status-interval", type=float, default=10.0, help="RoleStatus reporting interval in seconds (default: 10s)")
|
|
|
+
|
|
|
+ args = parser.parse_args()
|
|
|
+ role = args.role
|
|
|
+ use_cloud = args.config is None
|
|
|
+ config_path = args.config or f"config/config_{role}.json"
|
|
|
+ node_id = os.getenv("NODE_ID", "node01")
|
|
|
+ config_key = f"{role}_config:{node_id}"
|
|
|
+ status_key = f"vs:status:{role}:{node_id}"
|
|
|
+
|
|
|
+ logger = setup_app_logger(role.title())
|
|
|
+ logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
|
|
|
+
|
|
|
+ redis_client = SafeRedisClient(REDIS_CFG, logger.info)
|
|
|
+
|
|
|
+ # 启动云端指令实时监听(响应云端下发的升级指令)
|
|
|
+ start_upgrade_listener(REDIS_CFG, node_id, logger)
|
|
|
+
|
|
|
+ groups_conf, current_hash, _ = load_remote_or_cache(
|
|
|
+ config_key=config_key,
|
|
|
+ local_cache_path=config_path,
|
|
|
+ use_cloud=use_cloud,
|
|
|
+ logger=logger.info,
|
|
|
+ )
|
|
|
+
|
|
|
+ wrappers: Dict[str, GCOWrapper] = {}
|
|
|
+ gco_classes: Dict[str, type] = {}
|
|
|
+
|
|
|
+ sync_groups(role, groups_conf, wrappers, gco_classes, logger)
|
|
|
+ logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
|
|
|
+
|
|
|
+ # 打印本地已安装插件列表与初始 RoleStatus
|
|
|
+ initial_status = collect_role_status(role, node_id, wrappers)
|
|
|
+ status_dict = initial_status.to_dict()
|
|
|
+ logger.info("📊 Initial RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
|
|
|
+
|
|
|
+ report_ttl = max(30, int(args.status_interval * 3))
|
|
|
+ try:
|
|
|
+ redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
|
|
|
+ except Exception as e:
|
|
|
+ logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
|
|
|
+
|
|
|
+ if use_cloud:
|
|
|
+ logger.info(f"Cloud config active for key '{config_key}'. Polling every {args.poll_interval}s.")
|
|
|
+
|
|
|
+ last_poll_time = time.time()
|
|
|
+ last_status_report_time = time.time()
|
|
|
+
|
|
|
+ try:
|
|
|
+ while True:
|
|
|
+ time.sleep(1)
|
|
|
+ now = time.time()
|
|
|
+
|
|
|
+ # 1. 响应云端下发的即时升级指令 (PubSub)
|
|
|
+ while not upgrade_queue.empty():
|
|
|
+ try:
|
|
|
+ payload = upgrade_queue.get_nowait()
|
|
|
+ plugins_items = payload.get("plugins", [])
|
|
|
+ if isinstance(plugins_items, dict):
|
|
|
+ plugins_items = [plugins_items]
|
|
|
+ if plugins_items:
|
|
|
+ if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
|
|
|
+ # 升级成功,立即刷新并上报最新 RoleStatus
|
|
|
+ new_status = collect_role_status(role, node_id, wrappers)
|
|
|
+ status_dict = new_status.to_dict()
|
|
|
+ redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
|
|
|
+ logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
|
|
|
+ except queue.Empty:
|
|
|
+ break
|
|
|
+
|
|
|
+ # 2. 定时上报包含 installed_plugins 的 RoleStatus
|
|
|
+ if now - last_status_report_time >= args.status_interval:
|
|
|
+ last_status_report_time = now
|
|
|
+ try:
|
|
|
+ current_status = collect_role_status(role, node_id, wrappers)
|
|
|
+ status_dict = current_status.to_dict()
|
|
|
+ redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
|
|
|
+ logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
|
|
|
+ except Exception as e:
|
|
|
+ logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
|
|
|
+
|
|
|
+ # 3. 云端分组配置定时轮询与热更新
|
|
|
+ if use_cloud and (now - last_poll_time >= args.poll_interval):
|
|
|
+ last_poll_time = now
|
|
|
+ try:
|
|
|
+ new_conf = fetch_cloud_config(config_key)
|
|
|
+ if new_conf is not None:
|
|
|
+ new_hash = compute_config_hash(new_conf)
|
|
|
+ if new_hash != current_hash:
|
|
|
+ logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
|
|
|
+ save_local_cache(config_path, new_conf)
|
|
|
+ sync_groups(role, new_conf, wrappers, gco_classes, logger)
|
|
|
+ current_hash = new_hash
|
|
|
+ logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
|
|
|
+ except Exception as e:
|
|
|
+ logger.warning(f"Error polling cloud config key '{config_key}': {e}")
|
|
|
+
|
|
|
+ except KeyboardInterrupt:
|
|
|
+ logger.info(f"Shutting down {role}...")
|
|
|
+ for gid, wrapper in list(wrappers.items()):
|
|
|
+ try:
|
|
|
+ wrapper.stop()
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"Error stopping group [{gid}]: {e}")
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ main()
|