main.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. import os
  2. import sys
  3. import time
  4. import json
  5. import queue
  6. import argparse
  7. import threading
  8. from typing import Dict, Any, List
  9. from core_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
  10. from gco_wrapper import GCOWrapper
  11. from logger_setup import setup_app_logger
  12. import configure
  13. from configure import REDIS_CFG
  14. from sentinel import SentinelGCO
  15. from booker import BuiltinBookerGCO, OrderBookerGCO
  16. from standalone import StandaloneGCO
  17. from utils.safe_redis_cli import SafeRedisClient
  18. from utils.cloud_config import (
  19. load_remote_or_cache,
  20. compute_config_hash,
  21. fetch_cloud_config,
  22. save_local_cache,
  23. )
  24. from utils.plugin_updater import (
  25. get_installed_plugins,
  26. get_installed_plugin_metas,
  27. is_newer_version,
  28. download_and_install_plugin,
  29. )
  30. from utils.redis_stream_logger import RedisStreamLogger, RedisStreamLogHandler
  31. # Upgrade command queue (populated by Redis PubSub listener)
  32. upgrade_queue = queue.Queue()
  33. def get_gco_class(role: str, cfg: Any):
  34. """Determines specific GCO implementation class based on role and configuration."""
  35. if role == "sentinel":
  36. return SentinelGCO
  37. if role == "standalone":
  38. return StandaloneGCO
  39. if role == "booker":
  40. if cfg.booker.account_source == "order":
  41. return OrderBookerGCO
  42. return BuiltinBookerGCO
  43. raise ValueError(f"Unknown role: {role}")
  44. def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
  45. """Uniformly parses group configs into dict {identifier: config_obj}."""
  46. items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
  47. cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
  48. result = {}
  49. for item in items:
  50. cfg = cfg_cls.from_json(item)
  51. result[cfg.identifier] = cfg
  52. return result
  53. def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
  54. """Collects aggregated status of all GroupWrappers on current node and metadata of locally installed plugins."""
  55. groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
  56. installed = get_installed_plugin_metas()
  57. return RoleStatus(
  58. role=role,
  59. node_id=node_id,
  60. installed_plugins=installed,
  61. groups=groups_status,
  62. collected_at=utc_now_str(),
  63. )
  64. def reload_wrappers_for_plugin(
  65. plugin_name: str,
  66. wrappers: Dict[str, GCOWrapper],
  67. gco_classes: Dict[str, type],
  68. logger,
  69. ):
  70. """Finds wrappers utilizing the plugin and dynamically hot-reloads them."""
  71. for gid, wrapper in list(wrappers.items()):
  72. if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
  73. logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
  74. try:
  75. wrapper.stop()
  76. except Exception as e:
  77. logger.error(f"Error stopping group [{gid}] during reload: {e}")
  78. # Clear old module cache to load new code
  79. sys.modules.pop(plugin_name, None)
  80. try:
  81. target_cls = gco_classes[gid]
  82. new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
  83. new_wrapper.load()
  84. new_wrapper.start()
  85. wrappers[gid] = new_wrapper
  86. logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
  87. except Exception as e:
  88. logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
  89. def handle_plugin_upgrades(
  90. upgrade_items: List[Dict[str, Any]],
  91. wrappers: Dict[str, GCOWrapper],
  92. gco_classes: Dict[str, type],
  93. logger,
  94. ) -> bool:
  95. """
  96. Checks and executes plugin upgrade list.
  97. Supported format:
  98. [
  99. {
  100. "plugin_name": "tls_plugin",
  101. "version": "1.1.0",
  102. "download_url": "https://...", # or "code": "..."
  103. "filename": "tls_plugin.py"
  104. }
  105. ]
  106. """
  107. installed = get_installed_plugins()
  108. upgraded_any = False
  109. for item in upgrade_items:
  110. plugin_name = item.get("name")
  111. target_version = str(item.get("version", ""))
  112. if not plugin_name or not target_version:
  113. continue
  114. current_version = installed.get(plugin_name, "0.0.0")
  115. if is_newer_version(target_version, current_version):
  116. logger.info(
  117. f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
  118. )
  119. try:
  120. download_and_install_plugin(
  121. plugin_name=plugin_name,
  122. version=target_version,
  123. download_url=item.get("download_url"),
  124. code_content=item.get("code"),
  125. filename=item.get("filename"),
  126. expected_sha256=item.get("sha256"),
  127. )
  128. logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
  129. reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
  130. upgraded_any = True
  131. installed[plugin_name] = target_version
  132. except Exception as e:
  133. logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
  134. return upgraded_any
  135. def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
  136. """Starts Redis PubSub thread to listen for instant upgrade commands dispatched from cloud."""
  137. def _listen():
  138. import redis
  139. while True:
  140. try:
  141. r = redis.Redis(**redis_conf)
  142. pubsub = r.pubsub()
  143. channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
  144. pubsub.subscribe(*channels)
  145. logger.info(f"Subscribed to upgrade channels: {channels}")
  146. for msg in pubsub.listen():
  147. if msg and msg.get("type") == "message":
  148. try:
  149. raw_data = msg.get("data")
  150. if isinstance(raw_data, bytes):
  151. raw_data = raw_data.decode("utf-8")
  152. payload = json.loads(raw_data)
  153. logger.info(f"Received cloud upgrade command: {payload}")
  154. upgrade_queue.put(payload)
  155. except Exception as e:
  156. logger.error(f"Error parsing upgrade message: {e}")
  157. except Exception as e:
  158. logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
  159. time.sleep(5)
  160. threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
  161. def sync_groups(
  162. role: str,
  163. raw_conf: Any,
  164. wrappers: Dict[str, GCOWrapper],
  165. gco_classes: Dict[str, type],
  166. logger,
  167. ):
  168. """Synchronizes GCOWrapper lifecycle (stop, restart, update, start) according to latest config."""
  169. new_groups = parse_group_configs(role, raw_conf)
  170. # 1. Stop removed or disabled groups
  171. for gid in list(wrappers.keys()):
  172. cfg = new_groups.get(gid)
  173. if not cfg or not cfg.enable:
  174. logger.info(f"Stopping disabled/removed group [{gid}]...")
  175. try:
  176. wrappers[gid].stop()
  177. except Exception as e:
  178. logger.error(f"Error stopping group [{gid}]: {e}")
  179. del wrappers[gid]
  180. gco_classes.pop(gid, None)
  181. # 2. Update existing groups or start new groups
  182. for gid, cfg in new_groups.items():
  183. if not cfg.enable:
  184. continue
  185. target_cls = get_gco_class(role, cfg)
  186. if gid in wrappers:
  187. if gco_classes.get(gid) != target_cls:
  188. logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
  189. try:
  190. wrappers[gid].stop()
  191. except Exception as e:
  192. logger.error(f"Error stopping group [{gid}]: {e}")
  193. try:
  194. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  195. wrapper.load()
  196. wrapper.start()
  197. wrappers[gid] = wrapper
  198. gco_classes[gid] = target_cls
  199. except Exception as e:
  200. logger.error(f"Failed to restart group [{gid}]: {e}")
  201. else:
  202. logger.info(f"Updating configuration for group [{gid}]...")
  203. try:
  204. wrappers[gid].update_config(cfg)
  205. except Exception as e:
  206. logger.error(f"Error updating config for group [{gid}]: {e}")
  207. else:
  208. logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
  209. try:
  210. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  211. wrapper.load()
  212. wrapper.start()
  213. wrappers[gid] = wrapper
  214. gco_classes[gid] = target_cls
  215. except Exception as e:
  216. logger.error(f"Failed to start group [{gid}]: {e}")
  217. def main():
  218. parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
  219. parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
  220. parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
  221. parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
  222. parser.add_argument("--status-interval", type=float, default=60.0, help="RoleStatus reporting interval in seconds (default: 60s)")
  223. args = parser.parse_args()
  224. role = args.role
  225. use_cloud = args.config is None
  226. config_path = args.config or f"config/config_{role}.yaml"
  227. node_id = getattr(configure, "NODE_ID", None) or os.getenv("NODE_ID", "node01")
  228. status_key = f"vs:status:{role}:{node_id}"
  229. # Initialize Redis Stream logger and attach to main Logger
  230. stream_logger = RedisStreamLogger.initialize(
  231. redis_conf=REDIS_CFG,
  232. node_id=node_id,
  233. role=role,
  234. maxlen=2000,
  235. )
  236. stream_handler = RedisStreamLogHandler(stream_logger)
  237. logger = setup_app_logger(role.title(), extra_handlers=[stream_handler])
  238. logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
  239. redis_client = SafeRedisClient(REDIS_CFG, logger.info)
  240. # Start real-time cloud command listener (handles upgrade commands)
  241. start_upgrade_listener(REDIS_CFG, node_id, logger)
  242. groups_conf, current_hash, _ = load_remote_or_cache(
  243. role=role,
  244. node_id=node_id,
  245. local_cache_path=config_path,
  246. use_cloud=use_cloud,
  247. logger=logger.info,
  248. )
  249. wrappers: Dict[str, GCOWrapper] = {}
  250. gco_classes: Dict[str, type] = {}
  251. sync_groups(role, groups_conf, wrappers, gco_classes, logger)
  252. logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
  253. # Print locally installed plugins and initial RoleStatus summary
  254. initial_status = collect_role_status(role, node_id, wrappers)
  255. installed_count = len(initial_status.installed_plugins)
  256. active_count = len(wrappers)
  257. logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
  258. report_ttl = max(180, int(args.status_interval * 3))
  259. try:
  260. redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
  261. except Exception as e:
  262. logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
  263. if use_cloud:
  264. logger.info(f"Cloud config active for role '{role}' (node_id: '{node_id}'). Polling every {args.poll_interval}s.")
  265. last_poll_time = time.time()
  266. last_status_report_time = time.time()
  267. try:
  268. while True:
  269. time.sleep(1)
  270. now = time.time()
  271. # 1. Process instant upgrade commands from cloud (PubSub)
  272. while not upgrade_queue.empty():
  273. try:
  274. payload = upgrade_queue.get_nowait()
  275. plugins_items = []
  276. if isinstance(payload, list):
  277. plugins_items = payload
  278. elif isinstance(payload, dict):
  279. if "plugins" in payload:
  280. p = payload["plugins"]
  281. plugins_items = p if isinstance(p, list) else [p]
  282. else:
  283. plugins_items = [payload]
  284. if plugins_items:
  285. if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
  286. # Upgrade succeeded; immediately refresh and report latest RoleStatus
  287. new_status = collect_role_status(role, node_id, wrappers)
  288. status_dict = new_status.to_dict()
  289. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  290. logger.info(f"📊 Reported updated RoleStatus: {len(new_status.groups)} group(s), {len(new_status.installed_plugins)} plugin(s).")
  291. except queue.Empty:
  292. break
  293. # 2. Periodically report RoleStatus containing installed_plugins
  294. if now - last_status_report_time >= args.status_interval:
  295. last_status_report_time = now
  296. try:
  297. current_status = collect_role_status(role, node_id, wrappers)
  298. status_dict = current_status.to_dict()
  299. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  300. logger.info(f"📊 Reported RoleStatus: {len(current_status.groups)} group(s), {len(current_status.installed_plugins)} plugin(s).")
  301. except Exception as e:
  302. logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
  303. # 3. Periodically poll cloud group configuration for hot updates
  304. if use_cloud and (now - last_poll_time >= args.poll_interval):
  305. last_poll_time = now
  306. try:
  307. new_conf = fetch_cloud_config(role=role, node_id=node_id)
  308. if new_conf is not None:
  309. new_hash = compute_config_hash(new_conf)
  310. if new_hash != current_hash:
  311. logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
  312. save_local_cache(config_path, new_conf)
  313. sync_groups(role, new_conf, wrappers, gco_classes, logger)
  314. current_hash = new_hash
  315. logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
  316. except Exception as e:
  317. logger.warning(f"Error polling cloud config for role '{role}' (node_id: '{node_id}'): {e}")
  318. except KeyboardInterrupt:
  319. logger.info(f"Shutting down {role}...")
  320. for gid, wrapper in list(wrappers.items()):
  321. try:
  322. wrapper.stop()
  323. except Exception as e:
  324. logger.error(f"Error stopping group [{gid}]: {e}")
  325. try:
  326. stream_logger.close()
  327. except Exception:
  328. pass
  329. if __name__ == "__main__":
  330. main()