sentinel.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime
  7. from typing import List, Dict, Callable
  8. from vs_types import GroupConfig, VSPlgConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str
  9. from vs_plg_factory import VSPlgFactory
  10. from toolkit.vs_cloud_api import VSCloudApi
  11. from toolkit.proxy_manager import ProxyManager
  12. from utils.safe_redis_cli import SafeRedisClient
  13. class SentinelGCO:
  14. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  15. self.m_cfg = cfg
  16. self.m_factory = VSPlgFactory()
  17. self.m_logger = logger
  18. self.m_tasks: List[Task] = []
  19. self.m_lock = threading.RLock()
  20. self.m_stop_event = threading.Event()
  21. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  22. self.m_last_group_query_time = 0.0
  23. self.m_spawn_cooldown_until = 0.0
  24. def _log(self, message):
  25. if self.m_logger:
  26. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  27. else:
  28. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  29. def _get_average_interval(self) -> float:
  30. """计算当前组平均的查询间隔(秒)"""
  31. mode = self.m_cfg.sentinel.query_wait.mode
  32. if mode == QueryWaitMode.Loop:
  33. return 1.0
  34. elif mode == QueryWaitMode.Fixed:
  35. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  36. elif mode == QueryWaitMode.Random:
  37. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  38. return 30.0
  39. def update_config(self, new_cfg: GroupConfig):
  40. """
  41. 动态更新配置
  42. """
  43. with self.m_lock:
  44. if self.m_cfg.enable and not new_cfg.enable:
  45. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  46. elif not self.m_cfg.enable and new_cfg.enable:
  47. self._log("Config dynamically updated: Group ENABLED.")
  48. else:
  49. self._log("Config dynamically updated: Parameters refreshed.")
  50. self.m_cfg = new_cfg
  51. def get_status(self) -> GroupStatus:
  52. with self.m_lock:
  53. tasks = list(self.m_tasks)
  54. gid = self.m_cfg.identifier
  55. enable = self.m_cfg.enable
  56. plugin_name = self.m_cfg.plugin_config.plugin_name
  57. plugins_status = []
  58. healthy_count = 0
  59. for t in tasks:
  60. if t and t.instance:
  61. st = t.instance.get_plugin_status()
  62. if st.is_healthy:
  63. healthy_count += 1
  64. plugins_status.append(st)
  65. return GroupStatus(
  66. group_id=gid,
  67. enable=enable,
  68. plugin_name=plugin_name,
  69. total_tasks=len(tasks),
  70. healthy_tasks=healthy_count,
  71. plugins=plugins_status,
  72. updated_at=utc_now_str(),
  73. )
  74. def start(self):
  75. if not self.m_cfg.enable:
  76. return
  77. self._log("Starting Sentinel...")
  78. plugin_name = self.m_cfg.plugin_config.plugin_name
  79. class_name = "".join(part.title() for part in plugin_name.split('_'))
  80. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  81. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  82. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  83. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  84. def stop(self):
  85. self._log("Stopping Sentinel...")
  86. self.m_stop_event.set()
  87. with self.m_lock:
  88. tasks_to_cleanup = list(self.m_tasks)
  89. self.m_tasks.clear()
  90. for task in tasks_to_cleanup:
  91. self._cleanup_task(task, "sentinel stopped")
  92. def _cleanup_task(self, task: Task, reason: str):
  93. try:
  94. if task and task.instance:
  95. self._log(f"Cleaning up sentinel instance. reason={reason}")
  96. task.instance.cleanup()
  97. except Exception as e:
  98. self._log(f"Cleanup failed. reason={reason}, error={e}")
  99. def _remove_task(self, task: Task, reason: str):
  100. removed = False
  101. with self.m_lock:
  102. if task in self.m_tasks:
  103. self.m_tasks.remove(task)
  104. removed = True
  105. if removed:
  106. self._cleanup_task(task, reason)
  107. def _get_redis_key(self, routing_key: str) -> str:
  108. return f"vs:signal:{routing_key}"
  109. def _is_within_active_hours(self) -> bool:
  110. """
  111. 判断当前是否在允许创建实例的时间段内
  112. """
  113. start_str = self.m_cfg.active_time_start
  114. end_str = self.m_cfg.active_time_end
  115. current_bj_time = datetime.utcnow().time()
  116. start_time = datetime.strptime(start_str, "%H:%M").time()
  117. end_time = datetime.strptime(end_str, "%H:%M").time()
  118. return start_time <= current_bj_time <= end_time
  119. def _monitor_loop(self):
  120. self._log("Monitor loop started.")
  121. self.m_last_group_query_time = 0.0
  122. while not self.m_stop_event.is_set():
  123. try:
  124. time.sleep(1)
  125. now = time.time()
  126. with self.m_lock:
  127. tasks_to_check = list(self.m_tasks)
  128. active_tasks = []
  129. dead_tasks = []
  130. for t in tasks_to_check:
  131. if t.instance.health_check():
  132. active_tasks.append(t)
  133. else:
  134. dead_tasks.append(t)
  135. if dead_tasks:
  136. with self.m_lock:
  137. current_tasks = list(self.m_tasks)
  138. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  139. for t in dead_tasks:
  140. if t in current_tasks:
  141. self._cleanup_task(t, "health check failed")
  142. else:
  143. with self.m_lock:
  144. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  145. if not active_tasks:
  146. continue
  147. avg_interval = self._get_average_interval()
  148. global_gap = max(1.0, avg_interval / len(active_tasks))
  149. active_tasks.sort(key=lambda x: x.next_run)
  150. for task in active_tasks:
  151. if now < task.next_run:
  152. continue
  153. if now - self.m_last_group_query_time < global_gap:
  154. break
  155. apt_types = self.m_cfg.appointment_types
  156. if not apt_types:
  157. continue
  158. weights = [float(item.weight) for item in apt_types]
  159. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  160. interval = 30
  161. mode = task.qw_cfg.mode
  162. if mode == QueryWaitMode.Loop:
  163. interval = 1
  164. elif mode == QueryWaitMode.Fixed:
  165. interval = task.qw_cfg.fixed_wait
  166. elif mode == QueryWaitMode.Random:
  167. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  168. self.m_last_group_query_time = now
  169. try:
  170. VSCloudApi.Instance().slot_refresh_start(
  171. apt_type.routing_key,
  172. country=apt_type.country,
  173. city=apt_type.city,
  174. visa_type=apt_type.visa_type
  175. )
  176. result = task.instance.query(apt_type)
  177. result.apt_type = apt_type
  178. if result.success:
  179. ttl = self.m_cfg.sentinel.signal_ttl
  180. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  181. payload = {
  182. "group_id": self.m_cfg.identifier,
  183. "apt_type": apt_type.model_dump(),
  184. "query_result": result.to_snapshot_payload(),
  185. "timestamp": time.time()
  186. }
  187. redis_key = self._get_redis_key(apt_type.routing_key)
  188. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  189. payload["query_result"]["website"] = self.m_cfg.website
  190. VSCloudApi.Instance().slot_snapshot_report(payload["query_result"])
  191. VSCloudApi.Instance().slot_refresh_success(apt_type.routing_key)
  192. except Exception as e:
  193. self._log(f"Query exception: {e}")
  194. VSCloudApi.Instance().slot_refresh_fail(apt_type.routing_key, error=str(e))
  195. finally:
  196. task.next_run = time.time() + interval
  197. break
  198. except Exception as e:
  199. self._log(f"Monitor loop error: {e}")
  200. def _creator_loop(self):
  201. self._log("Creator loop started.")
  202. while not self.m_stop_event.wait(1.0):
  203. try:
  204. if not self._is_within_active_hours():
  205. continue
  206. if time.time() < self.m_spawn_cooldown_until:
  207. continue
  208. with self.m_lock:
  209. current = len(self.m_tasks)
  210. target = self.m_cfg.sentinel.target_instances
  211. if current < target:
  212. self._spawn_sentinel_worker()
  213. except Exception as e:
  214. self._log(f'Creator loop exception: {e}')
  215. def _spawn_sentinel_worker(self):
  216. instance = None
  217. task = None
  218. success = False
  219. plg_cfg = None
  220. try:
  221. plg_cfg = VSPlgConfig()
  222. plg_cfg.debug = self.m_cfg.debug
  223. plg_cfg.free_config = self.m_cfg.free_config
  224. plg_cfg.session_max_life = self.m_cfg.session_max_life
  225. if not self.m_cfg.need_account:
  226. plg_cfg.account.id = 0
  227. plg_cfg.account.username = "Guest"
  228. else:
  229. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  230. plg_cfg.account = type(plg_cfg.account)(**acc)
  231. if self.m_cfg.need_proxy:
  232. proxy = ProxyManager.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  233. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  234. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  235. instance.set_log(self.m_logger)
  236. instance.set_config(plg_cfg)
  237. instance.create_session()
  238. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
  239. with self.m_lock:
  240. self.m_tasks.append(task)
  241. success = True
  242. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  243. except Exception as e:
  244. err_str = str(e)
  245. self.m_spawn_cooldown_until = time.time() + 30.0
  246. self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
  247. rate_limited_indicators = [
  248. "42901" in err_str,
  249. "Rate limited" in err_str
  250. ]
  251. if any(rate_limited_indicators):
  252. if plg_cfg and plg_cfg.account.username != "Guest":
  253. VSCloudApi.lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  254. finally:
  255. if not success:
  256. if task:
  257. self._remove_task(task, "spawn failed")
  258. elif instance is not None:
  259. try:
  260. instance.cleanup()
  261. except Exception:
  262. pass