sentinel.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. import os
  2. import time
  3. import json
  4. import random
  5. import threading
  6. from datetime import datetime
  7. from typing import List, Dict, Callable
  8. from vs_types import GroupConfig, VSPlgConfig, Task, QueryWaitMode, GroupStatus, PluginStatus, utc_now_str
  9. from vs_plg_factory import VSPlgFactory
  10. from toolkit.vs_cloud_api import VSCloudApi
  11. from utils.safe_redis_cli import SafeRedisClient
  12. class SentinelGCO:
  13. def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
  14. self.m_cfg = cfg
  15. self.m_factory = VSPlgFactory()
  16. self.m_logger = logger
  17. self.m_tasks: List[Task] = []
  18. self.m_lock = threading.RLock()
  19. self.m_stop_event = threading.Event()
  20. self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
  21. self.m_last_group_query_time = 0.0
  22. def _log(self, message):
  23. if self.m_logger:
  24. self.m_logger(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  25. else:
  26. print(f'[SENTINEL] [{self.m_cfg.identifier}] {message}')
  27. def _get_average_interval(self) -> float:
  28. """计算当前组平均的查询间隔(秒)"""
  29. mode = self.m_cfg.sentinel.query_wait.mode
  30. if mode == QueryWaitMode.Loop:
  31. return 1.0
  32. elif mode == QueryWaitMode.Fixed:
  33. return float(self.m_cfg.sentinel.query_wait.fixed_wait)
  34. elif mode == QueryWaitMode.Random:
  35. return (self.m_cfg.sentinel.query_wait.random_min + self.m_cfg.sentinel.query_wait.random_max) / 2.0
  36. return 30.0
  37. def update_config(self, new_cfg: GroupConfig):
  38. """
  39. 动态更新配置
  40. """
  41. with self.m_lock:
  42. if self.m_cfg.enable and not new_cfg.enable:
  43. self._log("Config dynamically updated: Group DISABLED. Will stop creating new tasks.")
  44. elif not self.m_cfg.enable and new_cfg.enable:
  45. self._log("Config dynamically updated: Group ENABLED.")
  46. else:
  47. self._log("Config dynamically updated: Parameters refreshed.")
  48. self.m_cfg = new_cfg
  49. def get_status(self) -> GroupStatus:
  50. with self.m_lock:
  51. tasks = list(self.m_tasks)
  52. gid = self.m_cfg.identifier
  53. enable = self.m_cfg.enable
  54. plugin_name = self.m_cfg.plugin_config.plugin_name
  55. plugins_status = []
  56. healthy_count = 0
  57. for t in tasks:
  58. if t and t.instance:
  59. st = t.instance.get_plugin_status()
  60. if st.is_healthy:
  61. healthy_count += 1
  62. plugins_status.append(st)
  63. return GroupStatus(
  64. group_id=gid,
  65. enable=enable,
  66. plugin_name=plugin_name,
  67. total_tasks=len(tasks),
  68. healthy_tasks=healthy_count,
  69. plugins=plugins_status,
  70. updated_at=utc_now_str(),
  71. )
  72. def start(self):
  73. if not self.m_cfg.enable:
  74. return
  75. self._log("Starting Sentinel...")
  76. plugin_name = self.m_cfg.plugin_config.plugin_name
  77. class_name = "".join(part.title() for part in plugin_name.split('_'))
  78. plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
  79. self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
  80. threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
  81. threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
  82. def stop(self):
  83. self._log("Stopping Sentinel...")
  84. self.m_stop_event.set()
  85. with self.m_lock:
  86. tasks_to_cleanup = list(self.m_tasks)
  87. self.m_tasks.clear()
  88. for task in tasks_to_cleanup:
  89. self._cleanup_task(task, "sentinel stopped")
  90. def _cleanup_task(self, task: Task, reason: str):
  91. try:
  92. if task and task.instance:
  93. self._log(f"Cleaning up sentinel instance. reason={reason}")
  94. task.instance.cleanup()
  95. except Exception as e:
  96. self._log(f"Cleanup failed. reason={reason}, error={e}")
  97. def _remove_task(self, task: Task, reason: str):
  98. removed = False
  99. with self.m_lock:
  100. if task in self.m_tasks:
  101. self.m_tasks.remove(task)
  102. removed = True
  103. if removed:
  104. self._cleanup_task(task, reason)
  105. def _get_redis_key(self, routing_key: str) -> str:
  106. return f"vs:signal:{routing_key}"
  107. def _is_within_active_hours(self) -> bool:
  108. """
  109. 判断当前是否在允许创建实例的时间段内
  110. """
  111. start_str = self.m_cfg.active_time_start
  112. end_str = self.m_cfg.active_time_end
  113. current_bj_time = datetime.utcnow().time()
  114. start_time = datetime.strptime(start_str, "%H:%M").time()
  115. end_time = datetime.strptime(end_str, "%H:%M").time()
  116. return start_time <= current_bj_time <= end_time
  117. def _monitor_loop(self):
  118. self._log("Monitor loop started.")
  119. self.m_last_group_query_time = 0.0
  120. while not self.m_stop_event.is_set():
  121. try:
  122. time.sleep(1)
  123. now = time.time()
  124. with self.m_lock:
  125. tasks_to_check = list(self.m_tasks)
  126. active_tasks = []
  127. dead_tasks = []
  128. for t in tasks_to_check:
  129. if t.instance.health_check():
  130. active_tasks.append(t)
  131. else:
  132. dead_tasks.append(t)
  133. if dead_tasks:
  134. with self.m_lock:
  135. current_tasks = list(self.m_tasks)
  136. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  137. for t in dead_tasks:
  138. if t in current_tasks:
  139. self._cleanup_task(t, "health check failed")
  140. else:
  141. with self.m_lock:
  142. self.m_tasks = [t for t in self.m_tasks if t in active_tasks]
  143. if not active_tasks:
  144. continue
  145. avg_interval = self._get_average_interval()
  146. global_gap = max(1.0, avg_interval / len(active_tasks))
  147. active_tasks.sort(key=lambda x: x.next_run)
  148. for task in active_tasks:
  149. if now < task.next_run:
  150. continue
  151. if now - self.m_last_group_query_time < global_gap:
  152. break
  153. apt_types = self.m_cfg.appointment_types
  154. if not apt_types:
  155. continue
  156. weights = [float(item.weight) for item in apt_types]
  157. apt_type = random.choices(apt_types, weights=weights, k=1)[0]
  158. interval = 30
  159. mode = task.qw_cfg.mode
  160. if mode == QueryWaitMode.Loop:
  161. interval = 1
  162. elif mode == QueryWaitMode.Fixed:
  163. interval = task.qw_cfg.fixed_wait
  164. elif mode == QueryWaitMode.Random:
  165. interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
  166. self.m_last_group_query_time = now
  167. try:
  168. VSCloudApi.Instance().slot_refresh_start(
  169. apt_type.routing_key,
  170. country=apt_type.country,
  171. city=apt_type.city,
  172. visa_type=apt_type.visa_type
  173. )
  174. result = task.instance.query(apt_type)
  175. result.apt_type = apt_type
  176. if result.success:
  177. ttl = self.m_cfg.sentinel.signal_ttl
  178. self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
  179. payload = {
  180. "group_id": self.m_cfg.identifier,
  181. "apt_type": apt_type.model_dump(),
  182. "query_result": result.to_snapshot_payload(),
  183. "timestamp": time.time()
  184. }
  185. redis_key = self._get_redis_key(apt_type.routing_key)
  186. self.redis_client.setex(redis_key, ttl, json.dumps(payload))
  187. payload["query_result"]["website"] = self.m_cfg.website
  188. VSCloudApi.Instance().slot_snapshot_report(payload["query_result"])
  189. VSCloudApi.Instance().slot_refresh_success(apt_type.routing_key)
  190. except Exception as e:
  191. self._log(f"Query exception: {e}")
  192. VSCloudApi.Instance().slot_refresh_fail(apt_type.routing_key, error=str(e))
  193. finally:
  194. task.next_run = time.time() + interval
  195. break
  196. except Exception as e:
  197. self._log(f"Monitor loop error: {e}")
  198. def _creator_loop(self):
  199. self._log("Creator loop started.")
  200. while not self.m_stop_event.wait(1.0):
  201. try:
  202. if not self._is_within_active_hours():
  203. continue
  204. with self.m_lock:
  205. current = len(self.m_tasks)
  206. target = self.m_cfg.sentinel.target_instances
  207. if current < target:
  208. self._spawn_sentinel_worker()
  209. except Exception as e:
  210. self._log(f'Creator loop exception: {e}')
  211. def _spawn_sentinel_worker(self):
  212. instance = None
  213. task = None
  214. success = False
  215. plg_cfg = None
  216. try:
  217. plg_cfg = VSPlgConfig()
  218. plg_cfg.debug = self.m_cfg.debug
  219. plg_cfg.free_config = self.m_cfg.free_config
  220. plg_cfg.session_max_life = self.m_cfg.session_max_life
  221. if not self.m_cfg.need_account:
  222. plg_cfg.account.id = 0
  223. plg_cfg.account.username = "Guest"
  224. else:
  225. acc = VSCloudApi.Instance().get_next_account(self.m_cfg.sentinel.account_pool_id, self.m_cfg.sentinel.account_cd)
  226. plg_cfg.account = type(plg_cfg.account)(**acc)
  227. if self.m_cfg.need_proxy:
  228. proxy = VSCloudApi.Instance().get_next_proxy(self.m_cfg.proxy_pool, self.m_cfg.proxy_cd)
  229. plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
  230. instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
  231. instance.set_log(self.m_logger)
  232. instance.set_config(plg_cfg)
  233. task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time() + 999999, book_allowed=False)
  234. with self.m_lock:
  235. self.m_tasks.append(task)
  236. instance.create_session()
  237. task.next_run = time.time()
  238. success = True
  239. self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
  240. except Exception as e:
  241. err_str = str(e)
  242. self._log(f"Spawn failed: {err_str}")
  243. rate_limited_indicators = [
  244. "42901" in err_str,
  245. "Rate limited" in err_str
  246. ]
  247. if any(rate_limited_indicators):
  248. if plg_cfg and plg_cfg.account.username != "Guest":
  249. VSCloudApi.lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
  250. finally:
  251. if not success:
  252. if task:
  253. self._remove_task(task, "spawn failed")
  254. elif instance is not None:
  255. instance.cleanup()