main.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415
  1. import os
  2. import sys
  3. import time
  4. import json
  5. import queue
  6. import argparse
  7. import signal
  8. import threading
  9. from typing import Dict, Any, List
  10. from core_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
  11. from gco_wrapper import GCOWrapper, State
  12. from logger_setup import setup_app_logger
  13. import configure
  14. from configure import REDIS_CFG
  15. from sentinel import SentinelGCO
  16. from booker import BuiltinBookerGCO, OrderBookerGCO
  17. from standalone import StandaloneGCO
  18. from utils.safe_redis_cli import SafeRedisClient
  19. from utils.cloud_config import (
  20. load_remote_or_cache,
  21. compute_config_hash,
  22. fetch_cloud_config,
  23. save_local_cache,
  24. )
  25. from utils.plugin_updater import (
  26. get_installed_plugins,
  27. get_installed_plugin_metas,
  28. is_newer_version,
  29. download_and_install_plugin,
  30. )
  31. from utils.redis_stream_logger import RedisStreamLogger, RedisStreamLogHandler
  32. # Upgrade command queue (populated by Redis PubSub listener)
  33. upgrade_queue = queue.Queue()
  34. def get_gco_class(role: str, cfg: Any):
  35. """Determines specific GCO implementation class based on role and configuration."""
  36. if role == "sentinel":
  37. return SentinelGCO
  38. if role == "standalone":
  39. return StandaloneGCO
  40. if role == "booker":
  41. if cfg.booker.account_source == "order":
  42. return OrderBookerGCO
  43. return BuiltinBookerGCO
  44. raise ValueError(f"Unknown role: {role}")
  45. def parse_group_configs(role: str, raw_conf: Any) -> Dict[str, Any]:
  46. """Uniformly parses group configs into dict {identifier: config_obj}."""
  47. items = raw_conf if isinstance(raw_conf, list) else ([raw_conf] if isinstance(raw_conf, dict) else [])
  48. cfg_cls = BookerStandaloneConfig if role == "standalone" else GroupConfig
  49. result = {}
  50. for item in items:
  51. cfg = cfg_cls.from_json(item)
  52. result[cfg.identifier] = cfg
  53. return result
  54. def collect_role_status(role: str, node_id: str, wrappers: Dict[str, GCOWrapper]) -> RoleStatus:
  55. """Collects aggregated status of all GroupWrappers on current node and metadata of locally installed plugins."""
  56. groups_status = [wrapper.get_status() for wrapper in wrappers.values()]
  57. installed = get_installed_plugin_metas()
  58. return RoleStatus(
  59. role=role,
  60. node_id=node_id,
  61. installed_plugins=installed,
  62. groups=groups_status,
  63. collected_at=utc_now_str(),
  64. )
  65. def reload_wrappers_for_plugin(
  66. plugin_name: str,
  67. wrappers: Dict[str, GCOWrapper],
  68. gco_classes: Dict[str, type],
  69. logger,
  70. ):
  71. """Finds wrappers utilizing the plugin and dynamically hot-reloads them."""
  72. for gid, wrapper in list(wrappers.items()):
  73. if wrapper.m_cfg.plugin_config.plugin_name == plugin_name:
  74. logger.info(f"Hot-reloading group [{gid}] for upgraded plugin '{plugin_name}'...")
  75. try:
  76. wrapper.stop()
  77. except Exception as e:
  78. logger.error(f"Error stopping group [{gid}] during reload: {e}")
  79. continue
  80. # Clear old module cache to load new code
  81. sys.modules.pop(plugin_name, None)
  82. try:
  83. target_cls = gco_classes[gid]
  84. new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
  85. new_wrapper.load()
  86. wrappers[gid] = new_wrapper
  87. new_wrapper.start()
  88. logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
  89. except Exception as e:
  90. logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
  91. def handle_plugin_upgrades(
  92. upgrade_items: List[Dict[str, Any]],
  93. wrappers: Dict[str, GCOWrapper],
  94. gco_classes: Dict[str, type],
  95. logger,
  96. ) -> bool:
  97. """
  98. Checks and executes plugin upgrade list.
  99. Supported format:
  100. [
  101. {
  102. "plugin_name": "tls_plugin",
  103. "version": "1.1.0",
  104. "download_url": "https://...", # or "code": "..."
  105. "filename": "tls_plugin.py"
  106. }
  107. ]
  108. """
  109. installed = get_installed_plugins()
  110. upgraded_any = False
  111. for item in upgrade_items:
  112. plugin_name = item.get("name")
  113. target_version = str(item.get("version", ""))
  114. if not plugin_name or not target_version:
  115. continue
  116. current_version = installed.get(plugin_name, "0.0.0")
  117. if is_newer_version(target_version, current_version):
  118. logger.info(
  119. f"🚀 Upgrading plugin '{plugin_name}' (v{current_version} -> v{target_version})..."
  120. )
  121. try:
  122. download_and_install_plugin(
  123. plugin_name=plugin_name,
  124. version=target_version,
  125. download_url=item.get("download_url"),
  126. code_content=item.get("code"),
  127. filename=item.get("filename"),
  128. expected_sha256=item.get("sha256"),
  129. )
  130. logger.info(f"✅ Plugin '{plugin_name}' v{target_version} installed successfully.")
  131. reload_wrappers_for_plugin(plugin_name, wrappers, gco_classes, logger)
  132. upgraded_any = True
  133. installed[plugin_name] = target_version
  134. except Exception as e:
  135. logger.error(f"❌ Plugin '{plugin_name}' upgrade failed: {e}")
  136. return upgraded_any
  137. def start_upgrade_listener(redis_conf: Dict, node_id: str, logger, stop_event):
  138. """Cancellable PubSub listener; every reconnect closes the old connection."""
  139. def _listen():
  140. import redis
  141. while not stop_event.is_set():
  142. client = pubsub = None
  143. try:
  144. conf = dict(redis_conf)
  145. conf.update(socket_timeout=1, socket_connect_timeout=1)
  146. client = redis.Redis(**conf)
  147. pubsub = client.pubsub()
  148. pubsub.subscribe("vs:channel:upgrade", f"vs:channel:upgrade:{node_id}")
  149. while not stop_event.is_set():
  150. msg = pubsub.get_message(timeout=0.5)
  151. if msg and msg.get("type") == "message":
  152. try:
  153. payload = json.loads(msg["data"])
  154. upgrade_queue.put(payload)
  155. except Exception as exc:
  156. logger.error(f"Error parsing upgrade message: {exc}")
  157. except Exception as exc:
  158. if not stop_event.is_set():
  159. logger.warning(f"PubSub connection error: {exc}. Reconnecting in 5s...")
  160. finally:
  161. for resource in (pubsub, client):
  162. if resource is not None:
  163. try:
  164. resource.close()
  165. except Exception:
  166. pass
  167. stop_event.wait(5)
  168. thread = threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener")
  169. thread.start()
  170. return thread
  171. def shutdown_groups(wrappers, logger):
  172. """Stop groups concurrently so shutdown time does not grow with group count."""
  173. def stop(gid, wrapper):
  174. try:
  175. wrapper.stop()
  176. except Exception as exc:
  177. logger.error(f"Group [{gid}] cleanup pending: {exc}")
  178. threads = [threading.Thread(target=stop, args=(gid, wrapper), daemon=True)
  179. for gid, wrapper in list(wrappers.items())]
  180. for thread in threads:
  181. thread.start()
  182. timeout = max((w.m_cfg.shutdown_timeout + w.m_cfg.cleanup_timeout + 1
  183. for w in wrappers.values()), default=1)
  184. deadline = time.monotonic() + timeout
  185. for thread in threads:
  186. thread.join(max(0, deadline - time.monotonic()))
  187. def sync_groups(
  188. role: str,
  189. raw_conf: Any,
  190. wrappers: Dict[str, GCOWrapper],
  191. gco_classes: Dict[str, type],
  192. logger,
  193. ):
  194. """Synchronizes GCOWrapper lifecycle (stop, restart, update, start) according to latest config."""
  195. new_groups = parse_group_configs(role, raw_conf)
  196. # 1. Stop removed or disabled groups
  197. for gid in list(wrappers.keys()):
  198. cfg = new_groups.get(gid)
  199. if not cfg or not cfg.enable:
  200. logger.info(f"Stopping disabled/removed group [{gid}]...")
  201. try:
  202. wrappers[gid].stop()
  203. except Exception as e:
  204. logger.error(f"Error stopping group [{gid}]: {e}")
  205. continue
  206. del wrappers[gid]
  207. gco_classes.pop(gid, None)
  208. # 2. Update existing groups or start new groups
  209. for gid, cfg in new_groups.items():
  210. if not cfg.enable:
  211. continue
  212. target_cls = get_gco_class(role, cfg)
  213. if gid in wrappers:
  214. if gco_classes.get(gid) != target_cls or wrappers[gid]._state != State.RUNNING:
  215. logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
  216. try:
  217. wrappers[gid].stop()
  218. except Exception as e:
  219. logger.error(f"Error stopping group [{gid}]: {e}")
  220. continue
  221. try:
  222. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  223. wrapper.load()
  224. wrappers[gid] = wrapper
  225. gco_classes[gid] = target_cls
  226. wrapper.start()
  227. except Exception as e:
  228. logger.error(f"Failed to restart group [{gid}]: {e}")
  229. else:
  230. logger.info(f"Updating configuration for group [{gid}]...")
  231. try:
  232. wrappers[gid].update_config(cfg)
  233. except Exception as e:
  234. logger.error(f"Error updating config for group [{gid}]: {e}")
  235. else:
  236. logger.info(f"Starting group [{gid}] ({target_cls.__name__})...")
  237. try:
  238. wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
  239. wrapper.load()
  240. wrappers[gid] = wrapper
  241. gco_classes[gid] = target_cls
  242. wrapper.start()
  243. except Exception as e:
  244. logger.error(f"Failed to start group [{gid}]: {e}")
  245. def main():
  246. parser = argparse.ArgumentParser(description="Coordinate Unified Runner")
  247. parser.add_argument("role", choices=["sentinel", "booker", "standalone"], help="Role to run")
  248. parser.add_argument("-c", "--config", type=str, default=None, help="Path to local config file")
  249. parser.add_argument("--poll-interval", type=float, default=30.0, help="Cloud config poll interval in seconds")
  250. parser.add_argument("--status-interval", type=float, default=60.0, help="RoleStatus reporting interval in seconds (default: 60s)")
  251. args = parser.parse_args()
  252. role = args.role
  253. use_cloud = args.config is None
  254. config_path = args.config or f"config/config_{role}.yaml"
  255. node_id = getattr(configure, "NODE_ID", None) or os.getenv("NODE_ID", "node01")
  256. status_key = f"vs:status:{role}:{node_id}"
  257. # Initialize Redis Stream logger and attach to main Logger
  258. stream_logger = RedisStreamLogger.initialize(
  259. redis_conf=REDIS_CFG,
  260. node_id=node_id,
  261. role=role,
  262. maxlen=2000,
  263. )
  264. stream_handler = RedisStreamLogHandler(stream_logger)
  265. logger = setup_app_logger(role.title(), extra_handlers=[stream_handler])
  266. logger.info(f"{role.title()} Runner started (Node ID: '{node_id}', Cloud Mode: {use_cloud})")
  267. redis_client = SafeRedisClient(REDIS_CFG, logger.info)
  268. stop_event = threading.Event()
  269. previous_handlers = {}
  270. for signum in (signal.SIGINT, signal.SIGTERM):
  271. previous_handlers[signum] = signal.signal(signum, lambda *_: stop_event.set())
  272. wrappers: Dict[str, GCOWrapper] = {}
  273. gco_classes: Dict[str, type] = {}
  274. listener = None
  275. try:
  276. # Start real-time cloud command listener (handles upgrade commands)
  277. listener = start_upgrade_listener(REDIS_CFG, node_id, logger, stop_event)
  278. groups_conf, current_hash, _ = load_remote_or_cache(
  279. role=role,
  280. node_id=node_id,
  281. local_cache_path=config_path,
  282. use_cloud=use_cloud,
  283. logger=logger.info,
  284. )
  285. sync_groups(role, groups_conf, wrappers, gco_classes, logger)
  286. logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
  287. # Print locally installed plugins and initial RoleStatus summary
  288. initial_status = collect_role_status(role, node_id, wrappers)
  289. installed_count = len(initial_status.installed_plugins)
  290. active_count = len(wrappers)
  291. logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
  292. report_ttl = max(180, int(args.status_interval * 3))
  293. try:
  294. redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
  295. except Exception as e:
  296. logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
  297. if use_cloud:
  298. logger.info(f"Cloud config active for role '{role}' (node_id: '{node_id}'). Polling every {args.poll_interval}s.")
  299. last_poll_time = time.time()
  300. last_status_report_time = time.time()
  301. while not stop_event.wait(1):
  302. now = time.time()
  303. # 1. Process instant upgrade commands from cloud (PubSub)
  304. while not stop_event.is_set() and not upgrade_queue.empty():
  305. try:
  306. payload = upgrade_queue.get_nowait()
  307. plugins_items = []
  308. if isinstance(payload, list):
  309. plugins_items = payload
  310. elif isinstance(payload, dict):
  311. if "plugins" in payload:
  312. p = payload["plugins"]
  313. plugins_items = p if isinstance(p, list) else [p]
  314. else:
  315. plugins_items = [payload]
  316. if plugins_items:
  317. if handle_plugin_upgrades(plugins_items, wrappers, gco_classes, logger):
  318. # Upgrade succeeded; immediately refresh and report latest RoleStatus
  319. new_status = collect_role_status(role, node_id, wrappers)
  320. status_dict = new_status.to_dict()
  321. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  322. logger.info(f"📊 Reported updated RoleStatus: {len(new_status.groups)} group(s), {len(new_status.installed_plugins)} plugin(s).")
  323. except queue.Empty:
  324. break
  325. # 2. Periodically report RoleStatus containing installed_plugins
  326. if now - last_status_report_time >= args.status_interval:
  327. last_status_report_time = now
  328. try:
  329. current_status = collect_role_status(role, node_id, wrappers)
  330. status_dict = current_status.to_dict()
  331. redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
  332. logger.info(f"📊 Reported RoleStatus: {len(current_status.groups)} group(s), {len(current_status.installed_plugins)} plugin(s).")
  333. except Exception as e:
  334. logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
  335. # 3. Periodically poll cloud group configuration for hot updates
  336. if use_cloud and (now - last_poll_time >= args.poll_interval):
  337. last_poll_time = now
  338. try:
  339. new_conf = fetch_cloud_config(role=role, node_id=node_id)
  340. if new_conf is not None:
  341. new_hash = compute_config_hash(new_conf)
  342. if new_hash != current_hash:
  343. logger.info(f"Cloud config change detected ({current_hash[:8]} -> {new_hash[:8]}). Auto-reloading...")
  344. save_local_cache(config_path, new_conf)
  345. sync_groups(role, new_conf, wrappers, gco_classes, logger)
  346. current_hash = new_hash
  347. logger.info(f"Auto-reload completed. Currently {len(wrappers)} group(s) running.")
  348. except Exception as e:
  349. logger.warning(f"Error polling cloud config for role '{role}' (node_id: '{node_id}'): {e}")
  350. finally:
  351. stop_event.set()
  352. shutdown_groups(wrappers, logger)
  353. if listener is not None:
  354. listener.join(2)
  355. for resource in (redis_client, stream_logger):
  356. try:
  357. resource.close()
  358. except Exception as exc:
  359. logger.error(f'Runner connection cleanup failed: {exc}')
  360. for signum, handler in previous_handlers.items():
  361. signal.signal(signum, handler)
  362. if __name__ == "__main__":
  363. main()