plugin_utils.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490
  1. """
  2. plugin_utils.py - 签证插件通用基础工具模块
  3. 提供浏览器响应封装、Options/隧道初始化、JS Fetch 注入、截图、日期筛选、验证码求解及 Cookie 弹窗处理等通用函数
  4. """
  5. import os
  6. import json
  7. import time
  8. import socket
  9. import random
  10. import shutil
  11. from datetime import datetime
  12. from typing import List, Dict, Optional, Any, Callable, Tuple
  13. from urllib.parse import urlencode
  14. import configure
  15. from vs_types import BizLogicError, NotFoundError
  16. from toolkit.mihomo_tunnel import MihomoTunnel
  17. from utils.fingerprint_utils import FingerprintGenerator
  18. from DrissionPage import ChromiumOptions
  19. # ---------------------------------------------------------------------------
  20. # 1. 响应封装类
  21. # ---------------------------------------------------------------------------
  22. class BrowserResponse:
  23. """封装在浏览器上下文内执行 JS Fetch 返回的响应对象"""
  24. def __init__(self, result_dict: Optional[Dict[str, Any]] = None):
  25. result_dict = result_dict or {}
  26. self.status_code = result_dict.get('status', 0)
  27. self.text = result_dict.get('body', '')
  28. self.headers = result_dict.get('headers', {})
  29. self.url = result_dict.get('url', '')
  30. self._json = None
  31. def json(self) -> Dict[str, Any]:
  32. if self._json is None:
  33. if not self.text:
  34. return {}
  35. try:
  36. self._json = json.loads(self.text)
  37. except Exception:
  38. self._json = {}
  39. return self._json
  40. @property
  41. def content(self) -> bytes:
  42. return self.text.encode('utf-8')
  43. # ---------------------------------------------------------------------------
  44. # 2. 网络端口与浏览器启动选项
  45. # ---------------------------------------------------------------------------
  46. def get_free_port() -> int:
  47. """获取本地随机空闲 TCP 端口"""
  48. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
  49. s.bind(('', 0))
  50. return s.getsockname()[1]
  51. def init_chromium_options(
  52. user_data_path: str,
  53. username_seed: Optional[str] = None,
  54. proxy: Optional[Any] = None,
  55. headless: bool = False,
  56. window_size: str = "1920,1080",
  57. debug_port: Optional[int] = None,
  58. logger: Optional[Callable[[str], None]] = None
  59. ) -> Tuple[ChromiumOptions, Optional[MihomoTunnel]]:
  60. """
  61. 配置并返回 ChromiumOptions 以及(若启用了中转)创建好的 MihomoTunnel 实例
  62. :param user_data_path: 浏览器用户数据存储目录
  63. :param username_seed: 用于生成确定性指纹的账号特征种子
  64. :param proxy: 代理配置对象(含 ip, port, proto, username, password)
  65. :param headless: 是否使用无头模式
  66. :param window_size: 初始窗口尺寸 (宽,高)
  67. :param debug_port: 本地调试端口,不传则自动获取
  68. :param logger: 日志回调函数
  69. :return: (co, tunnel)
  70. """
  71. def _log(msg: str):
  72. if logger:
  73. logger(msg)
  74. co = ChromiumOptions()
  75. port = debug_port if debug_port is not None else get_free_port()
  76. co.set_local_port(port)
  77. co.set_user_data_path(user_data_path)
  78. chrome_path = configure.CHROME_PATH or os.getenv("CHROME_BIN")
  79. if chrome_path and os.path.exists(chrome_path):
  80. co.set_paths(browser_path=chrome_path)
  81. tunnel = None
  82. if proxy and getattr(proxy, 'ip', None):
  83. if getattr(proxy, 'username', None) and getattr(proxy, 'password', None):
  84. _log(f"Starting Proxy Tunnel for {proxy.ip}:{proxy.port}...")
  85. exit_node = {
  86. "name": "ExitNode",
  87. "type": proxy.proto,
  88. "server": proxy.ip,
  89. "port": proxy.port,
  90. "username": proxy.username,
  91. "password": proxy.password
  92. }
  93. relay_node = None
  94. if configure.MIHOMO_RELAY_NODES:
  95. relay_node = random.choice(configure.MIHOMO_RELAY_NODES)
  96. mihomo_path = configure.MIHOMO_BIN_PATH or os.getenv("MIHOMO_BIN")
  97. if not mihomo_path:
  98. raise BizLogicError(message='Mihomo path is null, You need set mihomo bin path in configure or os env')
  99. tunnel = MihomoTunnel(mihomo_path, exit_node=exit_node, relay_node=relay_node)
  100. local_proxy = tunnel.start()
  101. _log(f"Tunnel started at {local_proxy}")
  102. co.set_argument(f'--proxy-server={local_proxy}')
  103. else:
  104. proxy_str = f"{proxy.proto}://{proxy.ip}:{proxy.port}"
  105. co.set_argument(f'--proxy-server={proxy_str}')
  106. else:
  107. _log("[WARN] No proxy configured!")
  108. seed = username_seed or "default"
  109. specific_fp = FingerprintGenerator().generate(seed)
  110. fp_seed = specific_fp.get("seed")
  111. fp_platform = specific_fp.get("platform")
  112. fp_brand = specific_fp.get("brand")
  113. _log(f'Browser fingerprint seed={fp_seed}, platform={fp_platform}')
  114. co.headless(headless)
  115. co.set_argument('--no-sandbox')
  116. co.set_argument('--disable-dev-shm-usage')
  117. co.set_argument(f'--window-size={window_size}')
  118. co.set_argument('--disable-blink-features=AutomationControlled')
  119. co.set_argument(f"--fingerprint={fp_seed}")
  120. co.set_argument(f"--fingerprint-platform={fp_platform}")
  121. co.set_argument(f"--fingerprint-brand={fp_brand}")
  122. return co, tunnel
  123. # ---------------------------------------------------------------------------
  124. # 3. 浏览器内 JS Fetch 注入请求
  125. # ---------------------------------------------------------------------------
  126. def perform_browser_fetch(
  127. page,
  128. method: str,
  129. url: str,
  130. headers: Optional[Dict[str, str]] = None,
  131. data: Optional[Any] = None,
  132. json_data: Optional[Any] = None,
  133. params: Optional[Dict[str, Any]] = None,
  134. timeout: int = 30
  135. ) -> BrowserResponse:
  136. """
  137. 在 DrissionPage 浏览器页面上下文内注入并执行原生 fetch 请求
  138. """
  139. if not page:
  140. raise BizLogicError("Browser page is not initialized")
  141. if params:
  142. query_str = urlencode(params)
  143. url += ('&' if '?' in url else '?') + query_str
  144. fetch_options: Dict[str, Any] = {
  145. "method": method.upper(),
  146. "headers": headers or {},
  147. "credentials": "include"
  148. }
  149. if json_data is not None:
  150. fetch_options['body'] = json.dumps(json_data)
  151. fetch_options['headers']['Content-Type'] = 'application/json'
  152. elif data is not None:
  153. if isinstance(data, dict):
  154. fetch_options['body'] = urlencode(data)
  155. fetch_options['headers']['Content-Type'] = 'application/x-www-form-urlencoded'
  156. else:
  157. fetch_options['body'] = str(data)
  158. js_script = f"""
  159. const url = "{url}";
  160. const options = {json.dumps(fetch_options)};
  161. return fetch(url, options)
  162. .then(async response => {{
  163. const text = await response.text();
  164. const headers = {{}};
  165. response.headers.forEach((value, key) => headers[key] = value);
  166. return {{
  167. status: response.status,
  168. body: text,
  169. headers: headers,
  170. url: response.url
  171. }};
  172. }})
  173. .catch(error => {{
  174. return {{
  175. status: 0,
  176. body: error.toString(),
  177. headers: {{}},
  178. url: url
  179. }};
  180. }});
  181. """
  182. res_dict = page.run_js(js_script, timeout=timeout)
  183. return BrowserResponse(res_dict)
  184. # ---------------------------------------------------------------------------
  185. # 4. 辅助工具(截图、日期过滤、验证码求解、Cookie处理)
  186. # ---------------------------------------------------------------------------
  187. def save_screenshot(
  188. page,
  189. instance_id: str,
  190. name_prefix: str,
  191. save_dir: str = "data",
  192. full_page: bool = False,
  193. logger: Optional[Callable[[str], None]] = None
  194. ) -> Optional[str]:
  195. """保存页面截图至本地目录"""
  196. if not page:
  197. return None
  198. try:
  199. timestamp = int(time.time())
  200. filename = f"{instance_id}_{name_prefix}_{timestamp}.jpg"
  201. save_path = os.path.join(save_dir, filename)
  202. os.makedirs(save_dir, exist_ok=True)
  203. page.get_screenshot(path=save_path, full_page=full_page)
  204. if logger:
  205. logger(f"Screenshot saved to {save_path}")
  206. return save_path
  207. except Exception as e:
  208. if logger:
  209. logger(f"Failed to save screenshot: {e}")
  210. return None
  211. def filter_dates_in_range(
  212. dates: List[str],
  213. start_str: Optional[str] = None,
  214. end_str: Optional[str] = None,
  215. shuffle: bool = True
  216. ) -> List[str]:
  217. """
  218. 按起止日期范围筛选日期列表(格式: YYYY-MM-DD)
  219. """
  220. if not start_str or not end_str:
  221. return list(dates)
  222. valid_dates = []
  223. s_date = datetime.strptime(start_str[:10], "%Y-%m-%d")
  224. e_date = datetime.strptime(end_str[:10], "%Y-%m-%d")
  225. for date_str in dates:
  226. try:
  227. curr_date = datetime.strptime(date_str[:10], "%Y-%m-%d")
  228. if s_date <= curr_date <= e_date:
  229. valid_dates.append(date_str)
  230. except Exception:
  231. continue
  232. if shuffle:
  233. random.shuffle(valid_dates)
  234. return valid_dates
  235. def solve_capsolver_recaptcha(
  236. api_token: str,
  237. site_key: str,
  238. page_url: str,
  239. proxy: Optional[Any] = None,
  240. task_type: str = "ReCaptchaV2TaskProxyLess",
  241. action: Optional[str] = None
  242. ) -> str:
  243. """
  244. 调用 Capsolver API 异步求解 ReCaptchaV2
  245. """
  246. if not api_token:
  247. raise NotFoundError("Api-token required for capsolver")
  248. import requests as req
  249. submit_url = "https://api.capsolver.com/createTask"
  250. task: Dict[str, Any] = {
  251. "type": task_type,
  252. "websiteURL": page_url,
  253. "websiteKey": site_key,
  254. }
  255. if action:
  256. task["pageAction"] = action
  257. if proxy and proxy.ip and not task_type.endswith("ProxyLess"):
  258. task["proxyType"] = proxy.proto
  259. task["proxyAddress"] = proxy.ip
  260. task["proxyPort"] = proxy.port
  261. if getattr(proxy, 'username', None):
  262. task["proxyLogin"] = proxy.username
  263. task["proxyPassword"] = proxy.password
  264. payload = {"clientKey": api_token, "task": task}
  265. r = req.post(submit_url, json=payload, timeout=20)
  266. if r.status_code != 200:
  267. raise BizLogicError(message="Failed to submit capsolver task")
  268. task_id = r.json().get("taskId")
  269. for _ in range(20):
  270. res = req.post("https://api.capsolver.com/getTaskResult", json={"clientKey": api_token, "taskId": task_id}, timeout=20)
  271. if res.status_code == 200:
  272. d = res.json()
  273. if d.get("status") == "ready":
  274. return d["solution"]["gRecaptchaResponse"]
  275. time.sleep(3)
  276. raise BizLogicError(message="Capsolver task timeout")
  277. def solve_capsolver_hcaptcha(
  278. api_token: str,
  279. site_key: str,
  280. page_url: str,
  281. proxy: Optional[Any] = None,
  282. task_type: str = "HCaptchaTaskProxyLess",
  283. logger: Optional[Callable[[str], None]] = None
  284. ) -> str:
  285. """
  286. 调用 Capsolver API 异步求解 hCaptcha
  287. """
  288. if not api_token:
  289. raise NotFoundError("Capsolver API key is required")
  290. def _log(msg: str):
  291. if logger:
  292. logger(msg)
  293. import requests as req
  294. submit_url = "https://api.capsolver.com/createTask"
  295. task: Dict[str, Any] = {
  296. "type": task_type,
  297. "websiteURL": page_url,
  298. "websiteKey": site_key,
  299. }
  300. if proxy and getattr(proxy, 'ip', None):
  301. task["proxyType"] = proxy.proto
  302. task["proxyAddress"] = proxy.ip
  303. task["proxyPort"] = proxy.port
  304. if getattr(proxy, 'username', None):
  305. task["proxyLogin"] = proxy.username
  306. task["proxyPassword"] = proxy.password
  307. payload = {"clientKey": api_token, "task": task}
  308. _log("Submitting hCaptcha task to Capsolver...")
  309. r = req.post(submit_url, json=payload, timeout=20)
  310. if r.status_code != 200:
  311. raise BizLogicError(message=f"Failed to create Capsolver task: {r.text}")
  312. res_json = r.json()
  313. if res_json.get("errorId", 0) != 0:
  314. raise BizLogicError(message=f"Failed to create Capsolver task: {r.text}")
  315. task_id = res_json.get("taskId")
  316. if not task_id:
  317. raise BizLogicError(message=f"No taskId returned from Capsolver: {r.text}")
  318. _log(f"Waiting for hCaptcha solution (TaskId: {task_id})...")
  319. start_time = time.time()
  320. while time.time() - start_time < 120:
  321. time.sleep(3)
  322. res = req.post(
  323. "https://api.capsolver.com/getTaskResult",
  324. json={"clientKey": api_token, "taskId": task_id},
  325. timeout=20
  326. )
  327. if res.status_code == 200:
  328. result = res.json()
  329. status = result.get("status")
  330. if status == "ready":
  331. solution = result.get("solution", {})
  332. token = solution.get("gRecaptchaResponse") or solution.get("token") or solution.get("response")
  333. _log("✅ hCaptcha solved successfully.")
  334. return str(token)
  335. if status == "failed":
  336. raise BizLogicError(message=f"hCaptcha solve failed: {res.text}")
  337. raise BizLogicError(message="Capsolver task timed out (120s)")
  338. def handle_cookie_consent(
  339. page,
  340. mouse=None,
  341. timeout: int = 3,
  342. logger: Optional[Callable[[str], None]] = None
  343. ) -> bool:
  344. """
  345. 通用 Cookie Consent 弹窗识别与自动处理(支持 OneTrust、Osano 等常见弹窗)
  346. """
  347. if not page:
  348. return False
  349. # 1. 优先尝试 OneTrust 策略 (VFS 等站点)
  350. try:
  351. onetrust_js = """
  352. try {
  353. var acceptBtn = document.getElementById('onetrust-accept-btn-handler');
  354. if (acceptBtn) {
  355. acceptBtn.click();
  356. return 'clicked';
  357. }
  358. var banner = document.getElementById('onetrust-banner-sdk');
  359. if (banner) {
  360. banner.style.display = 'none';
  361. banner.remove();
  362. return 'removed';
  363. }
  364. } catch(e) {}
  365. return 'none';
  366. """
  367. ot_res = page.run_js(onetrust_js)
  368. if ot_res in ('clicked', 'removed'):
  369. if logger:
  370. logger(f"Handled OneTrust cookie banner ({ot_res}).")
  371. return True
  372. except Exception:
  373. pass
  374. # 2. 尝试 Osano 及 ARIA 属性弹窗 (TLS 等站点)
  375. try:
  376. dialog = page.ele('@aria-label=Cookie Consent Banner', timeout=timeout)
  377. if dialog:
  378. target_btn = dialog.ele('tag:button@text():Save') or dialog.ele('tag:button@text():Accept') or dialog.ele('tag:button@text():Accept all')
  379. if target_btn:
  380. if mouse:
  381. mouse.human_click_ele(target_btn)
  382. else:
  383. target_btn.click()
  384. if logger:
  385. logger("Handled cookie consent dialog successfully.")
  386. return True
  387. except Exception as e:
  388. if logger:
  389. logger(f"Exception handling cookie dialog: {e}")
  390. return False
  391. def cleanup_browser_workspace(
  392. page=None,
  393. root_workspace: Optional[str] = None,
  394. tunnel=None,
  395. logger: Optional[Callable[[str], None]] = None
  396. ) -> None:
  397. """
  398. 统一安全销毁浏览器实例、删除本地临时 workspace 目录并停止代理隧道
  399. """
  400. def _log(msg: str):
  401. if logger:
  402. logger(msg)
  403. # 1. 销毁浏览器
  404. if page:
  405. try:
  406. page.quit(force=True)
  407. except Exception:
  408. pass
  409. # 2. 清理临时用户数据目录 (带多次重试与容错)
  410. if root_workspace and os.path.exists(root_workspace):
  411. for _ in range(3):
  412. try:
  413. time.sleep(0.3)
  414. shutil.rmtree(root_workspace, ignore_errors=True)
  415. if not os.path.exists(root_workspace):
  416. break
  417. except BaseException:
  418. pass
  419. try:
  420. time.sleep(0.3)
  421. except BaseException:
  422. pass
  423. # 3. 停止代理隧道
  424. if tunnel:
  425. try:
  426. tunnel.stop()
  427. except Exception:
  428. pass