main.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. import os
  2. import sys
  3. import time
  4. import json
  5. import queue
  6. import argparse
  7. import threading
  8. from typing import Dict, Any, List
  9. from core_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
  10. from gco_wrapper import GCOWrapper
  11. from logger_setup import setup_app_logger
  12. import configure
  13. from configure import REDIS_CFG
  14. from sentinel import SentinelGCO
  15. from booker import BuiltinBookerGCO, OrderBookerGCO
  16. from standalone import StandaloneGCO
  17. from utils.safe_redis_cli import SafeRedisClient
  18. from utils.cloud_config import (
  19. load_remote_or_cache,
  20. compute_config_hash,
  21. fetch_cloud_config,
  22. save_local_cache,
  23. )
  24. from utils.plugin_updater import (
  25. get_installed_plugins,
  26. get_installed_plugin_metas,
  27. is_newer_version,
  28. download_and_install_plugin,
  29. )
  30. # 升级指令队列(由 Redis PubSub 线程推送)
  31. upgrade_queue = queue.Queue()
  32. def get_gco_class(role: str, cfg: Any):
  33. """根据角色与配置决定具体的 GCO 实现类"""
  34. if role == "sentinel":
  35. return SentinelGCO
  36. if role == "standalone":
  37. return StandaloneGCO
  38. if role == "booker":
  39. if cfg.booker.account_source == "order":
  40. return OrderBookerGCO
  41. return BuiltinBookerGCO
  42. raise ValueError(f"Unknown role: {role}")
  43. def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
  44. """统一解析组配置为字典 {identifier: config_obj}"""
  45. items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
  46. cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
  47. result = {}
  48. for item in items:
  49. cfg = cfg_cls.from_json(item)
  50. result[cfg.identifier] = cfg
  51. return result
  52. def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
  53. """收集当前节点下所有 GroupWrapper 的聚合状态及本地已安装插件完整元数据列表"""
  54. groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
  55. installed = get_installed_plugin_metas()
  56. return RoleStatus(
  57. role=role,
  58. node_id=node_id,
  59. installed_plugins=installed,
  60. groups=groups_status,
  61. collected_at=utc_now_str(),
  62. )
  63. def reload_wrappers_for_plugin(
  64. plugin_name: str,
  65. wrappers: Dict[str, GCOWrapper],
  66. gco_classes: Dict[str, type],
  67. logger,
  68. ):
  69. """找到正在使用该插件的 Wrapper 并进行动态热重载"""
  70. for gid, wrapper in list(wrappers.items()):
  71. if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
  72. logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
  73. try:
  74. wrapper.stop()
  75. except Exception as e:
  76. logger.error(f"Error stopping group [{gid}] during reload: {e}")
  77. # 清除旧模块缓存以加载最新代码
  78. sys.modules.pop(plugin_name, None)
  79. try:
  80. target_cls = gco_classes[gid]
  81. new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
  82. new_wrapper.load()
  83. new_wrapper.start()
  84. wrappers[gid] = new_wrapper
  85. logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
  86. except Exception as e:
  87. logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
  88. def handle_plugin_upgrades(
  89. upgrade_items: List[Dict[str, Any]],
  90. wrappers: Dict[str, GCOWrapper],
  91. gco_classes: Dict[str, type],
  92. logger,
  93. ) -> bool:
  94. """
  95. 检查并执行插件升级列表
  96. 支持格式:
  97. [
  98. {
  99. "plugin_name": "tls_plugin",
  100. "version": "1.1.0",
  101. "download_url": "https://...", # 或 "code": "..."
  102. "filename": "tls_plugin.py"
  103. }
  104. ]
  105. """
  106. installed = get_installed_plugins()
  107. upgraded_any = False
  108. for item in upgrade_items:
  109. plugin_name = item.get("name")
  110. target_version = str(item.get("version", ""))
  111. if not plugin_name or not target_version:
  112. continue
  113. current_version = installed.get(plugin_name, "0.0.0")
  114. if is_newer_version(target_version, current_version):
  115. logger.info(
  116. f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
  117. )
  118. try:
  119. download_and_install_plugin(
  120. plugin_name=plugin_name,
  121. version=target_version,
  122. download_url=item.get("download_url"),
  123. code_content=item.get("code"),
  124. filename=item.get("filename"),
  125. expected_sha256=item.get("sha256"),
  126. )
  127. logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
  128. reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
  129. upgraded_any = True
  130. installed[plugin_name] = target_version
  131. except Exception as e:
  132. logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
  133. return upgraded_any
  134. def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
  135. """启动 Redis PubSub 后台线程,秒级监听云端下发的即时升级指令"""
  136. def _listen():
  137. import redis
  138. while True:
  139. try:
  140. r = redis.Redis(**redis_conf)
  141. pubsub = r.pubsub()
  142. channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
  143. pubsub.subscribe(*channels)
  144. logger.info(f"Subscribed to upgrade channels: {channels}")
  145. for msg in pubsub.listen():
  146. if msg and msg.get("type") == "message":
  147. try:
  148. raw_data = msg.get("data")
  149. if isinstance(raw_data, bytes):
  150. raw_data = raw_data.decode("utf-8")
  151. payload = json.loads(raw_data)
  152. logger.info(f"Received cloud upgrade command: {payload}")
  153. upgrade_queue.put(payload)
  154. except Exception as e:
  155. logger.error(f"Error parsing upgrade message: {e}")
  156. except Exception as e:
  157. logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
  158. time.sleep(5)
  159. threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
  160. def sync_groups(
  161. role: str,
  162. raw_conf: Any,
  163. wrappers: Dict[str, GCOWrapper],
  164. gco_classes: Dict[str, type],
  165. logger,
  166. ):
  167. """根据最新配置同步 GCOWrapper 实例的生命周期(停止、重启、更新、新增)"""
  168. new_groups = parse_group_configs(role, raw_conf)
  169. # 1. 停止已被移除或禁用的组
  170. for gid in list(wrappers.keys()):
  171. cfg = new_groups.get(gid)
  172. if not cfg or not cfg.enable:
  173. logger.info(f"Stopping disabled/removed group [{gid}]...")
  174. try:
  175. wrappers[gid].stop()
  176. except Exception as e:
  177. logger.error(f"Error stopping group [{gid}]: {e}")
  178. del wrappers[gid]
  179. gco_classes.pop(gid, None)
  180. # 2. 处理现有组更新或新增组
  181. for gid, cfg in new_groups.items():
  182. if not cfg.enable:
  183. continue
  184. target_cls = get_gco_class(role, cfg)
  185. if gid in wrappers:
  186. if gco_classes.get(gid) != target_cls:
  187. logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
  188. try:
  189. wrappers[gid].stop()
  190. except Exception as e:
  191. logger.error(f"Error stopping group [{gid}]: {e}")
  192. try:
  193. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  194. wrapper.load()
  195. wrapper.start()
  196. wrappers[gid] = wrapper
  197. gco_classes[gid] = target_cls
  198. except Exception as e:
  199. logger.error(f"Failed to restart group [{gid}]: {e}")
  200. else:
  201. logger.info(f"Updating configuration for group [{gid}]...")
  202. try:
  203. wrappers[gid].update_config(cfg)
  204. except Exception as e:
  205. logger.error(f"Error updating config for group [{gid}]: {e}")
  206. else:
  207. logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
  208. try:
  209. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  210. wrapper.load()
  211. wrapper.start()
  212. wrappers[gid] = wrapper
  213. gco_classes[gid] = target_cls
  214. except Exception as e:
  215. logger.error(f"Failed to start group [{gid}]: {e}")
  216. def main():
  217. parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
  218. parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
  219. parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
  220. parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
  221. parser.add_argument("--status-interval", type=float, default=60.0, help="RoleStatus reporting interval in seconds (default: 60s)")
  222. args = parser.parse_args()
  223. role = args.role
  224. use_cloud = args.config is None
  225. config_path = args.config or f"config/config_{role}.json"
  226. node_id = getattr(configure, "NODE_ID", None) or os.getenv("NODE_ID", "node01")
  227. status_key = f"vs:status:{role}:{node_id}"
  228. logger = setup_app_logger(role.title())
  229. logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
  230. redis_client = SafeRedisClient(REDIS_CFG, logger.info)
  231. # 启动云端指令实时监听(响应云端下发的升级指令)
  232. start_upgrade_listener(REDIS_CFG, node_id, logger)
  233. groups_conf, current_hash, _ = load_remote_or_cache(
  234. role=role,
  235. node_id=node_id,
  236. local_cache_path=config_path,
  237. use_cloud=use_cloud,
  238. logger=logger.info,
  239. )
  240. wrappers: Dict[str, GCOWrapper] = {}
  241. gco_classes: Dict[str, type] = {}
  242. sync_groups(role, groups_conf, wrappers, gco_classes, logger)
  243. logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
  244. # 打印本地已安装插件列表与初始 RoleStatus 摘要
  245. initial_status = collect_role_status(role, node_id, wrappers)
  246. installed_count = len(initial_status.installed_plugins)
  247. active_count = len(wrappers)
  248. logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
  249. report_ttl = max(180, int(args.status_interval * 3))
  250. try:
  251. redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
  252. except Exception as e:
  253. logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
  254. if use_cloud:
  255. logger.info(f"Cloud config active for role '{role}' (node_id: '{node_id}'). Polling every {args.poll_interval}s.")
  256. last_poll_time = time.time()
  257. last_status_report_time = time.time()
  258. try:
  259. while True:
  260. time.sleep(1)
  261. now = time.time()
  262. # 1. 响应云端下发的即时升级指令 (PubSub)
  263. while not upgrade_queue.empty():
  264. try:
  265. payload = upgrade_queue.get_nowait()
  266. plugins_items = []
  267. if isinstance(payload, list):
  268. plugins_items = payload
  269. elif isinstance(payload, dict):
  270. if "plugins" in payload:
  271. p = payload["plugins"]
  272. plugins_items = p if isinstance(p, list) else [p]
  273. else:
  274. plugins_items = [payload]
  275. if plugins_items:
  276. if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
  277. # 升级成功,立即刷新并上报最新 RoleStatus
  278. new_status = collect_role_status(role, node_id, wrappers)
  279. status_dict = new_status.to_dict()
  280. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  281. logger.info(f"📊 Reported updated RoleStatus: {len(new_status.groups)} group(s), {len(new_status.installed_plugins)} plugin(s).")
  282. except queue.Empty:
  283. break
  284. # 2. 定时上报包含 installed_plugins 的 RoleStatus
  285. if now - last_status_report_time >= args.status_interval:
  286. last_status_report_time = now
  287. try:
  288. current_status = collect_role_status(role, node_id, wrappers)
  289. status_dict = current_status.to_dict()
  290. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  291. logger.info(f"📊 Reported RoleStatus: {len(current_status.groups)} group(s), {len(current_status.installed_plugins)} plugin(s).")
  292. except Exception as e:
  293. logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
  294. # 3. 云端分组配置定时轮询与热更新
  295. if use_cloud and (now - last_poll_time >= args.poll_interval):
  296. last_poll_time = now
  297. try:
  298. new_conf = fetch_cloud_config(role=role, node_id=node_id)
  299. if new_conf is not None:
  300. new_hash = compute_config_hash(new_conf)
  301. if new_hash != current_hash:
  302. logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
  303. save_local_cache(config_path, new_conf)
  304. sync_groups(role, new_conf, wrappers, gco_classes, logger)
  305. current_hash = new_hash
  306. logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
  307. except Exception as e:
  308. logger.warning(f"Error polling cloud config for role '{role}' (node_id: '{node_id}'): {e}")
  309. except KeyboardInterrupt:
  310. logger.info(f"Shutting down {role}...")
  311. for gid, wrapper in list(wrappers.items()):
  312. try:
  313. wrapper.stop()
  314. except Exception as e:
  315. logger.error(f"Error stopping group [{gid}]: {e}")
  316. if __name__ == "__main__":
  317. main()