sentinel.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime
  7. from typing import List, Dict, Callable
  8. from vs_types import GroupConfig, VSPlgConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str
  9. from vs_plg_factory import VSPlgFactory
  10. from toolkit.vs_cloud_api import VSCloudApi
  11. from utils.safe_redis_cli import SafeRedisClient
  12. class SentinelGCO:
  13. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  14. self.m_cfg = cfg
  15. self.m_factory = VSPlgFactory()
  16. self.m_logger = logger
  17. self.m_tasks: List[Task] = []
  18. self.m_lock = threading.RLock()
  19. self.m_stop_event = threading.Event()
  20. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  21. self.m_last_group_query_time = 0.0
  22. self.m_spawn_cooldown_until = 0.0
  23. def _log(self, message):
  24. if self.m_logger:
  25. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  26. else:
  27. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  28. def _get_average_interval(self) -> float:
  29. """计算当前组平均的查询间隔(秒)"""
  30. mode = self.m_cfg.sentinel.query_wait.mode
  31. if mode == QueryWaitMode.Loop:
  32. return 1.0
  33. elif mode == QueryWaitMode.Fixed:
  34. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  35. elif mode == QueryWaitMode.Random:
  36. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  37. return 30.0
  38. def update_config(self, new_cfg: GroupConfig):
  39. """
  40. 动态更新配置
  41. """
  42. with self.m_lock:
  43. if self.m_cfg.enable and not new_cfg.enable:
  44. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  45. elif not self.m_cfg.enable and new_cfg.enable:
  46. self._log("Config dynamically updated: Group ENABLED.")
  47. else:
  48. self._log("Config dynamically updated: Parameters refreshed.")
  49. self.m_cfg = new_cfg
  50. def get_status(self) -> GroupStatus:
  51. with self.m_lock:
  52. tasks = list(self.m_tasks)
  53. gid = self.m_cfg.identifier
  54. enable = self.m_cfg.enable
  55. plugin_name = self.m_cfg.plugin_config.plugin_name
  56. plugins_status = []
  57. healthy_count = 0
  58. for t in tasks:
  59. if t and t.instance:
  60. st = t.instance.get_plugin_status()
  61. if st.is_healthy:
  62. healthy_count += 1
  63. plugins_status.append(st)
  64. return GroupStatus(
  65. group_id=gid,
  66. enable=enable,
  67. plugin_name=plugin_name,
  68. total_tasks=len(tasks),
  69. healthy_tasks=healthy_count,
  70. plugins=plugins_status,
  71. updated_at=utc_now_str(),
  72. )
  73. def start(self):
  74. if not self.m_cfg.enable:
  75. return
  76. self._log("Starting Sentinel...")
  77. plugin_name = self.m_cfg.plugin_config.plugin_name
  78. class_name = "".join(part.title() for part in plugin_name.split('_'))
  79. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  80. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  81. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  82. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  83. def stop(self):
  84. self._log("Stopping Sentinel...")
  85. self.m_stop_event.set()
  86. with self.m_lock:
  87. tasks_to_cleanup = list(self.m_tasks)
  88. self.m_tasks.clear()
  89. for task in tasks_to_cleanup:
  90. self._cleanup_task(task, "sentinel stopped")
  91. def _cleanup_task(self, task: Task, reason: str):
  92. try:
  93. if task and task.instance:
  94. self._log(f"Cleaning up sentinel instance. reason={reason}")
  95. task.instance.cleanup()
  96. except Exception as e:
  97. self._log(f"Cleanup failed. reason={reason}, error={e}")
  98. def _remove_task(self, task: Task, reason: str):
  99. removed = False
  100. with self.m_lock:
  101. if task in self.m_tasks:
  102. self.m_tasks.remove(task)
  103. removed = True
  104. if removed:
  105. self._cleanup_task(task, reason)
  106. def _get_redis_key(self, routing_key: str) -> str:
  107. return f"vs:signal:{routing_key}"
  108. def _is_within_active_hours(self) -> bool:
  109. """
  110. 判断当前是否在允许创建实例的时间段内
  111. """
  112. start_str = self.m_cfg.active_time_start
  113. end_str = self.m_cfg.active_time_end
  114. current_bj_time = datetime.utcnow().time()
  115. start_time = datetime.strptime(start_str, "%H:%M").time()
  116. end_time = datetime.strptime(end_str, "%H:%M").time()
  117. return start_time <= current_bj_time <= end_time
  118. def _monitor_loop(self):
  119. self._log("Monitor loop started.")
  120. self.m_last_group_query_time = 0.0
  121. while not self.m_stop_event.is_set():
  122. try:
  123. time.sleep(1)
  124. now = time.time()
  125. with self.m_lock:
  126. tasks_to_check = list(self.m_tasks)
  127. active_tasks = []
  128. dead_tasks = []
  129. for t in tasks_to_check:
  130. if t.instance.health_check():
  131. active_tasks.append(t)
  132. else:
  133. dead_tasks.append(t)
  134. if dead_tasks:
  135. with self.m_lock:
  136. current_tasks = list(self.m_tasks)
  137. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  138. for t in dead_tasks:
  139. if t in current_tasks:
  140. self._cleanup_task(t, "health check failed")
  141. else:
  142. with self.m_lock:
  143. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  144. if not active_tasks:
  145. continue
  146. avg_interval = self._get_average_interval()
  147. global_gap = max(1.0, avg_interval / len(active_tasks))
  148. active_tasks.sort(key=lambda x: x.next_run)
  149. for task in active_tasks:
  150. if now < task.next_run:
  151. continue
  152. if now - self.m_last_group_query_time < global_gap:
  153. break
  154. apt_types = self.m_cfg.appointment_types
  155. if not apt_types:
  156. continue
  157. weights = [float(item.weight) for item in apt_types]
  158. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  159. interval = 30
  160. mode = task.qw_cfg.mode
  161. if mode == QueryWaitMode.Loop:
  162. interval = 1
  163. elif mode == QueryWaitMode.Fixed:
  164. interval = task.qw_cfg.fixed_wait
  165. elif mode == QueryWaitMode.Random:
  166. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  167. self.m_last_group_query_time = now
  168. try:
  169. VSCloudApi.Instance().slot_refresh_start(
  170. apt_type.routing_key,
  171. country=apt_type.country,
  172. city=apt_type.city,
  173. visa_type=apt_type.visa_type
  174. )
  175. result = task.instance.query(apt_type)
  176. result.apt_type = apt_type
  177. if result.success:
  178. ttl = self.m_cfg.sentinel.signal_ttl
  179. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  180. payload = {
  181. "group_id": self.m_cfg.identifier,
  182. "apt_type": apt_type.model_dump(),
  183. "query_result": result.to_snapshot_payload(),
  184. "timestamp": time.time()
  185. }
  186. redis_key = self._get_redis_key(apt_type.routing_key)
  187. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  188. payload["query_result"]["website"] = self.m_cfg.website
  189. VSCloudApi.Instance().slot_snapshot_report(payload["query_result"])
  190. VSCloudApi.Instance().slot_refresh_success(apt_type.routing_key)
  191. except Exception as e:
  192. self._log(f"Query exception: {e}")
  193. VSCloudApi.Instance().slot_refresh_fail(apt_type.routing_key, error=str(e))
  194. finally:
  195. task.next_run = time.time() + interval
  196. break
  197. except Exception as e:
  198. self._log(f"Monitor loop error: {e}")
  199. def _creator_loop(self):
  200. self._log("Creator loop started.")
  201. while not self.m_stop_event.wait(1.0):
  202. try:
  203. if not self._is_within_active_hours():
  204. continue
  205. if time.time() < self.m_spawn_cooldown_until:
  206. continue
  207. with self.m_lock:
  208. current = len(self.m_tasks)
  209. target = self.m_cfg.sentinel.target_instances
  210. if current < target:
  211. self._spawn_sentinel_worker()
  212. except Exception as e:
  213. self._log(f'Creator loop exception: {e}')
  214. def _spawn_sentinel_worker(self):
  215. instance = None
  216. task = None
  217. success = False
  218. plg_cfg = None
  219. try:
  220. plg_cfg = VSPlgConfig()
  221. plg_cfg.debug = self.m_cfg.debug
  222. plg_cfg.free_config = self.m_cfg.free_config
  223. plg_cfg.session_max_life = self.m_cfg.session_max_life
  224. if not self.m_cfg.need_account:
  225. plg_cfg.account.id = 0
  226. plg_cfg.account.username = "Guest"
  227. else:
  228. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  229. plg_cfg.account = type(plg_cfg.account)(**acc)
  230. if self.m_cfg.need_proxy:
  231. proxy = VSCloudApi.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  232. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  233. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  234. instance.set_log(self.m_logger)
  235. instance.set_config(plg_cfg)
  236. instance.create_session()
  237. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  238. with self.m_lock:
  239. self.m_tasks.append(task)
  240. success = True
  241. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  242. except Exception as e:
  243. err_str = str(e)
  244. self.m_spawn_cooldown_until = time.time() + 30.0
  245. self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
  246. rate_limited_indicators = [
  247. "42901" in err_str,
  248. "Rate limited" in err_str
  249. ]
  250. if any(rate_limited_indicators):
  251. if plg_cfg and plg_cfg.account.username != "Guest":
  252. VSCloudApi.lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  253. finally:
  254. if not success:
  255. if task:
  256. self._remove_task(task, "spawn failed")
  257. elif instance is not None:
  258. try:
  259. instance.cleanup()
  260. except Exception:
  261. pass