sentinel.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime
  7. from typing import List, Dict, Callable
  8. from vs_types import GroupConfig, VSPlgConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str
  9. from vs_plg_factory import VSPlgFactory
  10. from toolkit.vs_cloud_api import VSCloudApi
  11. from toolkit.proxy_manager import ProxyManager
  12. from toolkit.notifier import notify_date_changes
  13. from utils.safe_redis_cli import SafeRedisClient
  14. class SentinelGCO:
  15. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  16. self.m_cfg = cfg
  17. self.m_factory = VSPlgFactory()
  18. self.m_logger = logger
  19. self.m_tasks: List[Task] = []
  20. self.m_lock = threading.RLock()
  21. self.m_stop_event = threading.Event()
  22. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  23. self.m_last_group_query_time = 0.0
  24. self.m_spawn_cooldown_until = 0.0
  25. self.m_current_dates = {}
  26. def _log(self, message):
  27. if self.m_logger:
  28. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  29. else:
  30. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  31. def _get_average_interval(self) -> float:
  32. """计算当前组平均的查询间隔(秒)"""
  33. mode = self.m_cfg.sentinel.query_wait.mode
  34. if mode == QueryWaitMode.Loop:
  35. return 1.0
  36. elif mode == QueryWaitMode.Fixed:
  37. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  38. elif mode == QueryWaitMode.Random:
  39. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  40. return 30.0
  41. def update_config(self, new_cfg: GroupConfig):
  42. """
  43. 动态更新配置
  44. """
  45. with self.m_lock:
  46. if self.m_cfg.enable and not new_cfg.enable:
  47. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  48. elif not self.m_cfg.enable and new_cfg.enable:
  49. self._log("Config dynamically updated: Group ENABLED.")
  50. else:
  51. self._log("Config dynamically updated: Parameters refreshed.")
  52. self.m_cfg = new_cfg
  53. def get_status(self) -> GroupStatus:
  54. with self.m_lock:
  55. tasks = list(self.m_tasks)
  56. gid = self.m_cfg.identifier
  57. enable = self.m_cfg.enable
  58. plugin_name = self.m_cfg.plugin_config.plugin_name
  59. plugins_status = []
  60. healthy_count = 0
  61. for t in tasks:
  62. if t and t.instance:
  63. st = t.instance.get_plugin_status()
  64. if st.is_healthy:
  65. healthy_count += 1
  66. plugins_status.append(st)
  67. return GroupStatus(
  68. group_id=gid,
  69. enable=enable,
  70. plugin_name=plugin_name,
  71. total_tasks=len(tasks),
  72. healthy_tasks=healthy_count,
  73. plugins=plugins_status,
  74. updated_at=utc_now_str(),
  75. )
  76. def start(self):
  77. if not self.m_cfg.enable:
  78. return
  79. self._log("Starting Sentinel...")
  80. plugin_name = self.m_cfg.plugin_config.plugin_name
  81. class_name = "".join(part.title() for part in plugin_name.split('_'))
  82. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  83. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  84. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  85. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  86. def stop(self):
  87. self._log("Stopping Sentinel...")
  88. self.m_stop_event.set()
  89. with self.m_lock:
  90. tasks_to_cleanup = list(self.m_tasks)
  91. self.m_tasks.clear()
  92. for task in tasks_to_cleanup:
  93. self._cleanup_task(task, "sentinel stopped")
  94. def _cleanup_instance(self, instance, reason: str = ""):
  95. try:
  96. if instance:
  97. self._log(f"🧹 Cleaned up instance. Reason: {reason}")
  98. instance.cleanup()
  99. except Exception as e:
  100. self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
  101. def _cleanup_task(self, task: Task, reason: str):
  102. if task and task.instance:
  103. self._cleanup_instance(task.instance, reason)
  104. def _remove_task(self, task: Task, reason: str):
  105. removed = False
  106. with self.m_lock:
  107. if task in self.m_tasks:
  108. self.m_tasks.remove(task)
  109. removed = True
  110. if removed:
  111. self._cleanup_task(task, reason)
  112. def _get_redis_key(self, routing_key: str) -> str:
  113. return f"vs:signal:{routing_key}"
  114. def _is_within_active_hours(self) -> bool:
  115. """
  116. 判断当前是否在允许创建实例的时间段内
  117. """
  118. start_str = self.m_cfg.active_time_start
  119. end_str = self.m_cfg.active_time_end
  120. current_bj_time = datetime.utcnow().time()
  121. start_time = datetime.strptime(start_str, "%H:%M").time()
  122. end_time = datetime.strptime(end_str, "%H:%M").time()
  123. return start_time <= current_bj_time <= end_time
  124. def _monitor_loop(self):
  125. self._log("Monitor loop started.")
  126. self.m_last_group_query_time = 0.0
  127. while not self.m_stop_event.is_set():
  128. try:
  129. time.sleep(1)
  130. now = time.time()
  131. with self.m_lock:
  132. tasks_to_check = list(self.m_tasks)
  133. active_tasks = []
  134. dead_tasks = []
  135. for t in tasks_to_check:
  136. if t.instance.health_check():
  137. active_tasks.append(t)
  138. else:
  139. dead_tasks.append(t)
  140. if dead_tasks:
  141. with self.m_lock:
  142. current_tasks = list(self.m_tasks)
  143. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  144. for t in dead_tasks:
  145. if t in current_tasks:
  146. self._cleanup_task(t, "health check failed")
  147. else:
  148. with self.m_lock:
  149. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  150. if not active_tasks:
  151. continue
  152. avg_interval = self._get_average_interval()
  153. global_gap = max(1.0, avg_interval / len(active_tasks))
  154. active_tasks.sort(key=lambda x: x.next_run)
  155. for task in active_tasks:
  156. if now < task.next_run:
  157. continue
  158. if now - self.m_last_group_query_time < global_gap:
  159. break
  160. apt_types = self.m_cfg.appointment_types
  161. if not apt_types:
  162. continue
  163. weights = [max(0.0, float(item.weight)) for item in apt_types]
  164. if sum(weights) > 0:
  165. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  166. else:
  167. apt_type = random.choice(apt_types)
  168. interval = 30
  169. mode = task.qw_cfg.mode
  170. if mode == QueryWaitMode.Loop:
  171. interval = 1
  172. elif mode == QueryWaitMode.Fixed:
  173. interval = task.qw_cfg.fixed_wait
  174. elif mode == QueryWaitMode.Random:
  175. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  176. self.m_last_group_query_time = now
  177. try:
  178. result = task.instance.query(apt_type)
  179. result.apt_type = apt_type
  180. notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
  181. if result.success:
  182. ttl = self.m_cfg.sentinel.signal_ttl
  183. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  184. payload = {
  185. "group_id": self.m_cfg.identifier,
  186. "apt_type": apt_type.model_dump(),
  187. "query_result": result.to_snapshot_payload(),
  188. "timestamp": time.time()
  189. }
  190. redis_key = self._get_redis_key(apt_type.routing_key)
  191. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  192. except Exception as e:
  193. self._log(f"Query exception: {e}")
  194. finally:
  195. task.next_run = time.time() + interval
  196. break
  197. except Exception as e:
  198. self._log(f"Monitor loop error: {e}")
  199. def _creator_loop(self):
  200. self._log("Creator loop started.")
  201. while not self.m_stop_event.wait(1.0):
  202. if not self._is_within_active_hours():
  203. continue
  204. if time.time() < self.m_spawn_cooldown_until:
  205. continue
  206. with self.m_lock:
  207. current = len(self.m_tasks)
  208. target = self.m_cfg.sentinel.target_instances
  209. if current < target:
  210. self._spawn_sentinel_worker()
  211. def _spawn_sentinel_worker(self):
  212. instance = None
  213. task = None
  214. success = False
  215. plg_cfg = None
  216. try:
  217. plg_cfg = VSPlgConfig()
  218. plg_cfg.debug = self.m_cfg.debug
  219. plg_cfg.free_config = self.m_cfg.free_config
  220. plg_cfg.session_max_life = self.m_cfg.session_max_life
  221. if not self.m_cfg.need_account:
  222. plg_cfg.account.id = 0
  223. plg_cfg.account.username = "Guest"
  224. else:
  225. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  226. plg_cfg.account = type(plg_cfg.account)(**acc)
  227. if self.m_cfg.need_proxy:
  228. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  229. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  230. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  231. instance.set_log(self.m_logger)
  232. instance.set_config(plg_cfg)
  233. instance.create_session()
  234. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  235. with self.m_lock:
  236. self.m_tasks.append(task)
  237. success = True
  238. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  239. except Exception as e:
  240. err_str = str(e)
  241. self.m_spawn_cooldown_until = time.time() + 30.0
  242. self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
  243. rate_limited_indicators = [
  244. "42901" in err_str,
  245. "Rate limited" in err_str
  246. ]
  247. if any(rate_limited_indicators):
  248. if plg_cfg and plg_cfg.account.username != "Guest":
  249. VSCloudApi.Instance().lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  250. finally:
  251. if not success and instance:
  252. self._cleanup_instance(instance, "spawn failed")