main.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. import os
  2. import sys
  3. import time
  4. import json
  5. import queue
  6. import argparse
  7. import threading
  8. from typing import Dict, Any, List
  9. from core_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
  10. from gco_wrapper import GCOWrapper
  11. from logger_setup import setup_app_logger
  12. import configure
  13. from configure import REDIS_CFG
  14. from sentinel import SentinelGCO
  15. from booker import BuiltinBookerGCO, OrderBookerGCO
  16. from standalone import StandaloneGCO
  17. from utils.safe_redis_cli import SafeRedisClient
  18. from utils.cloud_config import (
  19. load_remote_or_cache,
  20. compute_config_hash,
  21. fetch_cloud_config,
  22. save_local_cache,
  23. )
  24. from utils.plugin_updater import (
  25. get_installed_plugins,
  26. get_installed_plugin_metas,
  27. is_newer_version,
  28. download_and_install_plugin,
  29. )
  30. from utils.redis_stream_logger import RedisStreamLogger, RedisStreamLogHandler
  31. # 升级指令队列(由 Redis PubSub 线程推送)
  32. upgrade_queue = queue.Queue()
  33. def get_gco_class(role: str, cfg: Any):
  34. """根据角色与配置决定具体的 GCO 实现类"""
  35. if role == "sentinel":
  36. return SentinelGCO
  37. if role == "standalone":
  38. return StandaloneGCO
  39. if role == "booker":
  40. if cfg.booker.account_source == "order":
  41. return OrderBookerGCO
  42. return BuiltinBookerGCO
  43. raise ValueError(f"Unknown role: {role}")
  44. def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
  45. """统一解析组配置为字典 {identifier: config_obj}"""
  46. items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
  47. cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
  48. result = {}
  49. for item in items:
  50. cfg = cfg_cls.from_json(item)
  51. result[cfg.identifier] = cfg
  52. return result
  53. def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
  54. """收集当前节点下所有 GroupWrapper 的聚合状态及本地已安装插件完整元数据列表"""
  55. groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
  56. installed = get_installed_plugin_metas()
  57. return RoleStatus(
  58. role=role,
  59. node_id=node_id,
  60. installed_plugins=installed,
  61. groups=groups_status,
  62. collected_at=utc_now_str(),
  63. )
  64. def reload_wrappers_for_plugin(
  65. plugin_name: str,
  66. wrappers: Dict[str, GCOWrapper],
  67. gco_classes: Dict[str, type],
  68. logger,
  69. ):
  70. """找到正在使用该插件的 Wrapper 并进行动态热重载"""
  71. for gid, wrapper in list(wrappers.items()):
  72. if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
  73. logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
  74. try:
  75. wrapper.stop()
  76. except Exception as e:
  77. logger.error(f"Error stopping group [{gid}] during reload: {e}")
  78. # 清除旧模块缓存以加载最新代码
  79. sys.modules.pop(plugin_name, None)
  80. try:
  81. target_cls = gco_classes[gid]
  82. new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
  83. new_wrapper.load()
  84. new_wrapper.start()
  85. wrappers[gid] = new_wrapper
  86. logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
  87. except Exception as e:
  88. logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
  89. def handle_plugin_upgrades(
  90. upgrade_items: List[Dict[str, Any]],
  91. wrappers: Dict[str, GCOWrapper],
  92. gco_classes: Dict[str, type],
  93. logger,
  94. ) -> bool:
  95. """
  96. 检查并执行插件升级列表
  97. 支持格式:
  98. [
  99. {
  100. "plugin_name": "tls_plugin",
  101. "version": "1.1.0",
  102. "download_url": "https://...", # 或 "code": "..."
  103. "filename": "tls_plugin.py"
  104. }
  105. ]
  106. """
  107. installed = get_installed_plugins()
  108. upgraded_any = False
  109. for item in upgrade_items:
  110. plugin_name = item.get("name")
  111. target_version = str(item.get("version", ""))
  112. if not plugin_name or not target_version:
  113. continue
  114. current_version = installed.get(plugin_name, "0.0.0")
  115. if is_newer_version(target_version, current_version):
  116. logger.info(
  117. f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
  118. )
  119. try:
  120. download_and_install_plugin(
  121. plugin_name=plugin_name,
  122. version=target_version,
  123. download_url=item.get("download_url"),
  124. code_content=item.get("code"),
  125. filename=item.get("filename"),
  126. expected_sha256=item.get("sha256"),
  127. )
  128. logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
  129. reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
  130. upgraded_any = True
  131. installed[plugin_name] = target_version
  132. except Exception as e:
  133. logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
  134. return upgraded_any
  135. def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
  136. """启动 Redis PubSub 后台线程,秒级监听云端下发的即时升级指令"""
  137. def _listen():
  138. import redis
  139. while True:
  140. try:
  141. r = redis.Redis(**redis_conf)
  142. pubsub = r.pubsub()
  143. channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
  144. pubsub.subscribe(*channels)
  145. logger.info(f"Subscribed to upgrade channels: {channels}")
  146. for msg in pubsub.listen():
  147. if msg and msg.get("type") == "message":
  148. try:
  149. raw_data = msg.get("data")
  150. if isinstance(raw_data, bytes):
  151. raw_data = raw_data.decode("utf-8")
  152. payload = json.loads(raw_data)
  153. logger.info(f"Received cloud upgrade command: {payload}")
  154. upgrade_queue.put(payload)
  155. except Exception as e:
  156. logger.error(f"Error parsing upgrade message: {e}")
  157. except Exception as e:
  158. logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
  159. time.sleep(5)
  160. threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
  161. def sync_groups(
  162. role: str,
  163. raw_conf: Any,
  164. wrappers: Dict[str, GCOWrapper],
  165. gco_classes: Dict[str, type],
  166. logger,
  167. ):
  168. """根据最新配置同步 GCOWrapper 实例的生命周期(停止、重启、更新、新增)"""
  169. new_groups = parse_group_configs(role, raw_conf)
  170. # 1. 停止已被移除或禁用的组
  171. for gid in list(wrappers.keys()):
  172. cfg = new_groups.get(gid)
  173. if not cfg or not cfg.enable:
  174. logger.info(f"Stopping disabled/removed group [{gid}]...")
  175. try:
  176. wrappers[gid].stop()
  177. except Exception as e:
  178. logger.error(f"Error stopping group [{gid}]: {e}")
  179. del wrappers[gid]
  180. gco_classes.pop(gid, None)
  181. # 2. 处理现有组更新或新增组
  182. for gid, cfg in new_groups.items():
  183. if not cfg.enable:
  184. continue
  185. target_cls = get_gco_class(role, cfg)
  186. if gid in wrappers:
  187. if gco_classes.get(gid) != target_cls:
  188. logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
  189. try:
  190. wrappers[gid].stop()
  191. except Exception as e:
  192. logger.error(f"Error stopping group [{gid}]: {e}")
  193. try:
  194. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  195. wrapper.load()
  196. wrapper.start()
  197. wrappers[gid] = wrapper
  198. gco_classes[gid] = target_cls
  199. except Exception as e:
  200. logger.error(f"Failed to restart group [{gid}]: {e}")
  201. else:
  202. logger.info(f"Updating configuration for group [{gid}]...")
  203. try:
  204. wrappers[gid].update_config(cfg)
  205. except Exception as e:
  206. logger.error(f"Error updating config for group [{gid}]: {e}")
  207. else:
  208. logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
  209. try:
  210. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  211. wrapper.load()
  212. wrapper.start()
  213. wrappers[gid] = wrapper
  214. gco_classes[gid] = target_cls
  215. except Exception as e:
  216. logger.error(f"Failed to start group [{gid}]: {e}")
  217. def main():
  218. parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
  219. parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
  220. parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
  221. parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
  222. parser.add_argument("--status-interval", type=float, default=60.0, help="RoleStatus reporting interval in seconds (default: 60s)")
  223. args = parser.parse_args()
  224. role = args.role
  225. use_cloud = args.config is None
  226. config_path = args.config or f"config/config_{role}.yaml"
  227. node_id = getattr(configure, "NODE_ID", None) or os.getenv("NODE_ID", "node01")
  228. status_key = f"vs:status:{role}:{node_id}"
  229. # 初始化 Redis Stream 日志器并挂载到主 Logger
  230. stream_logger = RedisStreamLogger.initialize(
  231. redis_conf=REDIS_CFG,
  232. node_id=node_id,
  233. role=role,
  234. maxlen=2000,
  235. )
  236. stream_handler = RedisStreamLogHandler(stream_logger)
  237. logger = setup_app_logger(role.title(), extra_handlers=[stream_handler])
  238. logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
  239. redis_client = SafeRedisClient(REDIS_CFG, logger.info)
  240. # 启动云端指令实时监听(响应云端下发的升级指令)
  241. start_upgrade_listener(REDIS_CFG, node_id, logger)
  242. groups_conf, current_hash, _ = load_remote_or_cache(
  243. role=role,
  244. node_id=node_id,
  245. local_cache_path=config_path,
  246. use_cloud=use_cloud,
  247. logger=logger.info,
  248. )
  249. wrappers: Dict[str, GCOWrapper] = {}
  250. gco_classes: Dict[str, type] = {}
  251. sync_groups(role, groups_conf, wrappers, gco_classes, logger)
  252. logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
  253. # 打印本地已安装插件列表与初始 RoleStatus 摘要
  254. initial_status = collect_role_status(role, node_id, wrappers)
  255. installed_count = len(initial_status.installed_plugins)
  256. active_count = len(wrappers)
  257. logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
  258. report_ttl = max(180, int(args.status_interval * 3))
  259. try:
  260. redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
  261. except Exception as e:
  262. logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
  263. if use_cloud:
  264. logger.info(f"Cloud config active for role '{role}' (node_id: '{node_id}'). Polling every {args.poll_interval}s.")
  265. last_poll_time = time.time()
  266. last_status_report_time = time.time()
  267. try:
  268. while True:
  269. time.sleep(1)
  270. now = time.time()
  271. # 1. 响应云端下发的即时升级指令 (PubSub)
  272. while not upgrade_queue.empty():
  273. try:
  274. payload = upgrade_queue.get_nowait()
  275. plugins_items = []
  276. if isinstance(payload, list):
  277. plugins_items = payload
  278. elif isinstance(payload, dict):
  279. if "plugins" in payload:
  280. p = payload["plugins"]
  281. plugins_items = p if isinstance(p, list) else [p]
  282. else:
  283. plugins_items = [payload]
  284. if plugins_items:
  285. if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
  286. # 升级成功,立即刷新并上报最新 RoleStatus
  287. new_status = collect_role_status(role, node_id, wrappers)
  288. status_dict = new_status.to_dict()
  289. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  290. logger.info(f"📊 Reported updated RoleStatus: {len(new_status.groups)} group(s), {len(new_status.installed_plugins)} plugin(s).")
  291. except queue.Empty:
  292. break
  293. # 2. 定时上报包含 installed_plugins 的 RoleStatus
  294. if now - last_status_report_time >= args.status_interval:
  295. last_status_report_time = now
  296. try:
  297. current_status = collect_role_status(role, node_id, wrappers)
  298. status_dict = current_status.to_dict()
  299. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  300. logger.info(f"📊 Reported RoleStatus: {len(current_status.groups)} group(s), {len(current_status.installed_plugins)} plugin(s).")
  301. except Exception as e:
  302. logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
  303. # 3. 云端分组配置定时轮询与热更新
  304. if use_cloud and (now - last_poll_time >= args.poll_interval):
  305. last_poll_time = now
  306. try:
  307. new_conf = fetch_cloud_config(role=role, node_id=node_id)
  308. if new_conf is not None:
  309. new_hash = compute_config_hash(new_conf)
  310. if new_hash != current_hash:
  311. logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
  312. save_local_cache(config_path, new_conf)
  313. sync_groups(role, new_conf, wrappers, gco_classes, logger)
  314. current_hash = new_hash
  315. logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
  316. except Exception as e:
  317. logger.warning(f"Error polling cloud config for role '{role}' (node_id: '{node_id}'): {e}")
  318. except KeyboardInterrupt:
  319. logger.info(f"Shutting down {role}...")
  320. for gid, wrapper in list(wrappers.items()):
  321. try:
  322. wrapper.stop()
  323. except Exception as e:
  324. logger.error(f"Error stopping group [{gid}]: {e}")
  325. try:
  326. stream_logger.close()
  327. except Exception:
  328. pass
  329. if __name__ == "__main__":
  330. main()