main.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. import os
  2. import sys
  3. import time
  4. import json
  5. import queue
  6. import argparse
  7. import threading
  8. from typing import Dict, Any, List
  9. from vs_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
  10. from gco_wrapper import GCOWrapper
  11. from logger_setup import setup_app_logger
  12. from configure import REDIS_CFG
  13. from sentinel import SentinelGCO
  14. from booker import BuiltinBookerGCO, OrderBookerGCO
  15. from standalone import StandaloneGCO
  16. from utils.safe_redis_cli import SafeRedisClient
  17. from utils.cloud_config import (
  18. load_remote_or_cache,
  19. compute_config_hash,
  20. fetch_cloud_config,
  21. save_local_cache,
  22. )
  23. from utils.plugin_updater import (
  24. get_installed_plugins,
  25. get_installed_plugin_metas,
  26. is_newer_version,
  27. download_and_install_plugin,
  28. )
  29. # 升级指令队列(由 Redis PubSub 线程推送)
  30. upgrade_queue = queue.Queue()
  31. def get_gco_class(role: str, cfg: Any):
  32. """根据角色与配置决定具体的 GCO 实现类"""
  33. if role == "sentinel":
  34. return SentinelGCO
  35. if role == "standalone":
  36. return StandaloneGCO
  37. if role == "booker":
  38. if cfg.booker.account_source == "order":
  39. return OrderBookerGCO
  40. return BuiltinBookerGCO
  41. raise ValueError(f"Unknown role: {role}")
  42. def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
  43. """统一解析组配置为字典 {identifier: config_obj}"""
  44. items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
  45. cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
  46. result = {}
  47. for item in items:
  48. cfg = cfg_cls.from_json(item)
  49. result[cfg.identifier] = cfg
  50. return result
  51. def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
  52. """收集当前节点下所有 GroupWrapper 的聚合状态及本地已安装插件完整元数据列表"""
  53. groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
  54. installed = get_installed_plugin_metas()
  55. return RoleStatus(
  56. role=role,
  57. node_id=node_id,
  58. installed_plugins=installed,
  59. groups=groups_status,
  60. collected_at=utc_now_str(),
  61. )
  62. def reload_wrappers_for_plugin(
  63. plugin_name: str,
  64. wrappers: Dict[str, GCOWrapper],
  65. gco_classes: Dict[str, type],
  66. logger,
  67. ):
  68. """找到正在使用该插件的 Wrapper 并进行动态热重载"""
  69. for gid, wrapper in list(wrappers.items()):
  70. if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
  71. logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
  72. try:
  73. wrapper.stop()
  74. except Exception as e:
  75. logger.error(f"Error stopping group [{gid}] during reload: {e}")
  76. # 清除旧模块缓存以加载最新代码
  77. sys.modules.pop(plugin_name, None)
  78. try:
  79. target_cls = gco_classes[gid]
  80. new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
  81. new_wrapper.load()
  82. new_wrapper.start()
  83. wrappers[gid] = new_wrapper
  84. logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
  85. except Exception as e:
  86. logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
  87. def handle_plugin_upgrades(
  88. upgrade_items: List[Dict[str, Any]],
  89. wrappers: Dict[str, GCOWrapper],
  90. gco_classes: Dict[str, type],
  91. logger,
  92. ) -> bool:
  93. """
  94. 检查并执行插件升级列表
  95. 支持格式:
  96. [
  97. {
  98. "plugin_name": "tls_plugin",
  99. "version": "1.1.0",
  100. "download_url": "https://...", # 或 "code": "..."
  101. "filename": "tls_plugin.py"
  102. }
  103. ]
  104. """
  105. installed = get_installed_plugins()
  106. upgraded_any = False
  107. for item in upgrade_items:
  108. plugin_name = item.get("plugin_name")
  109. target_version = str(item.get("version", ""))
  110. if not plugin_name or not target_version:
  111. continue
  112. current_version = installed.get(plugin_name, "0.0.0")
  113. if is_newer_version(target_version, current_version):
  114. logger.info(
  115. f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
  116. )
  117. try:
  118. download_and_install_plugin(
  119. plugin_name=plugin_name,
  120. version=target_version,
  121. download_url=item.get("download_url"),
  122. code_content=item.get("code"),
  123. filename=item.get("filename"),
  124. )
  125. logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
  126. reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
  127. upgraded_any = True
  128. installed[plugin_name] = target_version
  129. except Exception as e:
  130. logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
  131. return upgraded_any
  132. def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
  133. """启动 Redis PubSub 后台线程,秒级监听云端下发的即时升级指令"""
  134. def _listen():
  135. import redis
  136. while True:
  137. try:
  138. r = redis.Redis(**redis_conf)
  139. pubsub = r.pubsub()
  140. channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
  141. pubsub.subscribe(*channels)
  142. logger.info(f"Subscribed to upgrade channels: {channels}")
  143. for msg in pubsub.listen():
  144. if msg and msg.get("type") == "message":
  145. try:
  146. raw_data = msg.get("data")
  147. if isinstance(raw_data, bytes):
  148. raw_data = raw_data.decode("utf-8")
  149. payload = json.loads(raw_data)
  150. logger.info(f"Received cloud upgrade command: {payload}")
  151. upgrade_queue.put(payload)
  152. except Exception as e:
  153. logger.error(f"Error parsing upgrade message: {e}")
  154. except Exception as e:
  155. logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
  156. time.sleep(5)
  157. threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
  158. def sync_groups(
  159. role: str,
  160. raw_conf: Any,
  161. wrappers: Dict[str, GCOWrapper],
  162. gco_classes: Dict[str, type],
  163. logger,
  164. ):
  165. """根据最新配置同步 GCOWrapper 实例的生命周期(停止、重启、更新、新增)"""
  166. new_groups = parse_group_configs(role, raw_conf)
  167. # 1. 停止已被移除或禁用的组
  168. for gid in list(wrappers.keys()):
  169. cfg = new_groups.get(gid)
  170. if not cfg or not cfg.enable:
  171. logger.info(f"Stopping disabled/removed group [{gid}]...")
  172. try:
  173. wrappers[gid].stop()
  174. except Exception as e:
  175. logger.error(f"Error stopping group [{gid}]: {e}")
  176. del wrappers[gid]
  177. gco_classes.pop(gid, None)
  178. # 2. 处理现有组更新或新增组
  179. for gid, cfg in new_groups.items():
  180. if not cfg.enable:
  181. continue
  182. target_cls = get_gco_class(role, cfg)
  183. if gid in wrappers:
  184. if gco_classes.get(gid) != target_cls:
  185. logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
  186. try:
  187. wrappers[gid].stop()
  188. except Exception as e:
  189. logger.error(f"Error stopping group [{gid}]: {e}")
  190. try:
  191. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  192. wrapper.load()
  193. wrapper.start()
  194. wrappers[gid] = wrapper
  195. gco_classes[gid] = target_cls
  196. except Exception as e:
  197. logger.error(f"Failed to restart group [{gid}]: {e}")
  198. else:
  199. logger.info(f"Updating configuration for group [{gid}]...")
  200. try:
  201. wrappers[gid].update_config(cfg)
  202. except Exception as e:
  203. logger.error(f"Error updating config for group [{gid}]: {e}")
  204. else:
  205. logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
  206. try:
  207. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  208. wrapper.load()
  209. wrapper.start()
  210. wrappers[gid] = wrapper
  211. gco_classes[gid] = target_cls
  212. except Exception as e:
  213. logger.error(f"Failed to start group [{gid}]: {e}")
  214. def main():
  215. parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
  216. parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
  217. parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
  218. parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
  219. parser.add_argument("--status-interval", type=float, default=60.0, help="RoleStatus reporting interval in seconds (default: 60s)")
  220. args = parser.parse_args()
  221. role = args.role
  222. use_cloud = args.config is None
  223. config_path = args.config or f"config/config_{role}.json"
  224. node_id = os.getenv("NODE_ID", "node01")
  225. config_key = f"{role}_config:{node_id}"
  226. status_key = f"vs:status:{role}:{node_id}"
  227. logger = setup_app_logger(role.title())
  228. logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
  229. redis_client = SafeRedisClient(REDIS_CFG, logger.info)
  230. # 启动云端指令实时监听(响应云端下发的升级指令)
  231. start_upgrade_listener(REDIS_CFG, node_id, logger)
  232. groups_conf, current_hash, _ = load_remote_or_cache(
  233. config_key=config_key,
  234. local_cache_path=config_path,
  235. use_cloud=use_cloud,
  236. logger=logger.info,
  237. )
  238. wrappers: Dict[str, GCOWrapper] = {}
  239. gco_classes: Dict[str, type] = {}
  240. sync_groups(role, groups_conf, wrappers, gco_classes, logger)
  241. logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
  242. # 打印本地已安装插件列表与初始 RoleStatus
  243. initial_status = collect_role_status(role, node_id, wrappers)
  244. status_dict = initial_status.to_dict()
  245. logger.info("📊 Initial RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
  246. report_ttl = max(180, int(args.status_interval * 3))
  247. try:
  248. redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
  249. except Exception as e:
  250. logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
  251. if use_cloud:
  252. logger.info(f"Cloud config active for key '{config_key}'. Polling every {args.poll_interval}s.")
  253. last_poll_time = time.time()
  254. last_status_report_time = time.time()
  255. try:
  256. while True:
  257. time.sleep(1)
  258. now = time.time()
  259. # 1. 响应云端下发的即时升级指令 (PubSub)
  260. while not upgrade_queue.empty():
  261. try:
  262. payload = upgrade_queue.get_nowait()
  263. plugins_items = payload.get("plugins", [])
  264. if isinstance(plugins_items, dict):
  265. plugins_items = [plugins_items]
  266. if plugins_items:
  267. if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
  268. # 升级成功,立即刷新并上报最新 RoleStatus
  269. new_status = collect_role_status(role, node_id, wrappers)
  270. status_dict = new_status.to_dict()
  271. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  272. logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
  273. except queue.Empty:
  274. break
  275. # 2. 定时上报包含 installed_plugins 的 RoleStatus
  276. if now - last_status_report_time >= args.status_interval:
  277. last_status_report_time = now
  278. try:
  279. current_status = collect_role_status(role, node_id, wrappers)
  280. status_dict = current_status.to_dict()
  281. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  282. logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
  283. except Exception as e:
  284. logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
  285. # 3. 云端分组配置定时轮询与热更新
  286. if use_cloud and (now - last_poll_time >= args.poll_interval):
  287. last_poll_time = now
  288. try:
  289. new_conf = fetch_cloud_config(config_key)
  290. if new_conf is not None:
  291. new_hash = compute_config_hash(new_conf)
  292. if new_hash != current_hash:
  293. logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
  294. save_local_cache(config_path, new_conf)
  295. sync_groups(role, new_conf, wrappers, gco_classes, logger)
  296. current_hash = new_hash
  297. logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
  298. except Exception as e:
  299. logger.warning(f"Error polling cloud config key '{config_key}': {e}")
  300. except KeyboardInterrupt:
  301. logger.info(f"Shutting down {role}...")
  302. for gid, wrapper in list(wrappers.items()):
  303. try:
  304. wrapper.stop()
  305. except Exception as e:
  306. logger.error(f"Error stopping group [{gid}]: {e}")
  307. if __name__ == "__main__":
  308. main()