""" plugin_utils.py - 签证插件通用基础工具模块 提供浏览器响应封装、Options/隧道初始化、JS Fetch 注入、截图、日期筛选、验证码求解及 Cookie 弹窗处理等通用函数 """ import os import json import time import socket import random import shutil from datetime import datetime from typing import List, Dict, Optional, Any, Callable, Tuple from urllib.parse import urlencode import configure from core_types import BizLogicError, NotFoundError from toolkit.proxy_tunnel import MihomoTunnel from utils.fingerprint_utils import FingerprintGenerator from DrissionPage import ChromiumOptions # --------------------------------------------------------------------------- # 1. 响应封装类 # --------------------------------------------------------------------------- class BrowserResponse: """封装在浏览器上下文内执行 JS Fetch 返回的响应对象""" def __init__(self, result_dict: Optional[Dict[str, Any]] = None): result_dict = result_dict or {} self.status_code = result_dict.get('status', 0) self.text = result_dict.get('body', '') self.headers = result_dict.get('headers', {}) self.url = result_dict.get('url', '') self._json = None def json(self) -> Dict[str, Any]: if self._json is None: if not self.text: return {} try: self._json = json.loads(self.text) except Exception: self._json = {} return self._json @property def content(self) -> bytes: return self.text.encode('utf-8') # --------------------------------------------------------------------------- # 2. 网络端口与浏览器启动选项 # --------------------------------------------------------------------------- def get_free_port() -> int: """获取本地随机空闲 TCP 端口""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(('', 0)) return s.getsockname()[1] def init_chromium_options( user_data_path: str, username_seed: Optional[str] = None, proxy: Optional[Any] = None, headless: bool = False, window_size: str = "1920,1080", debug_port: Optional[int] = None, logger: Optional[Callable[[str], None]] = None ) -> Tuple[ChromiumOptions, Optional[MihomoTunnel]]: """ 配置并返回 ChromiumOptions 以及(若启用了中转)创建好的 MihomoTunnel 实例 :param user_data_path: 浏览器用户数据存储目录 :param username_seed: 用于生成确定性指纹的账号特征种子 :param proxy: 代理配置对象(含 ip, port, proto, username, password) :param headless: 是否使用无头模式 :param window_size: 初始窗口尺寸 (宽,高) :param debug_port: 本地调试端口,不传则自动获取 :param logger: 日志回调函数 :return: (co, tunnel) """ def _log(msg: str): if logger: logger(msg) co = ChromiumOptions() port = debug_port if debug_port is not None else get_free_port() co.set_local_port(port) co.set_user_data_path(user_data_path) chrome_path = configure.CHROME_PATH or os.getenv("CHROME_BIN") if chrome_path and os.path.exists(chrome_path): co.set_paths(browser_path=chrome_path) tunnel = None if proxy and getattr(proxy, 'ip', None): if getattr(proxy, 'username', None) and getattr(proxy, 'password', None): _log(f"Starting Proxy Tunnel for {proxy.ip}:{proxy.port}...") exit_node = { "name": "ExitNode", "type": proxy.proto, "server": proxy.ip, "port": proxy.port, "username": proxy.username, "password": proxy.password } relay_node = None if configure.MIHOMO_RELAY_NODES: relay_node = random.choice(configure.MIHOMO_RELAY_NODES) mihomo_path = configure.MIHOMO_BIN_PATH or os.getenv("MIHOMO_BIN") if not mihomo_path: raise BizLogicError(message='Mihomo path is null, You need set mihomo bin path in configure or os env') tunnel = MihomoTunnel(mihomo_path, exit_node=exit_node, relay_node=relay_node) local_proxy = tunnel.start() _log(f"Tunnel started at {local_proxy}") co.set_argument(f'--proxy-server={local_proxy}') else: proxy_str = f"{proxy.proto}://{proxy.ip}:{proxy.port}" co.set_argument(f'--proxy-server={proxy_str}') else: _log("[WARN] No proxy configured!") seed = username_seed or "default" specific_fp = FingerprintGenerator().generate(seed) fp_seed = specific_fp.get("seed") fp_platform = specific_fp.get("platform") fp_brand = specific_fp.get("brand") _log(f'Browser fingerprint seed={fp_seed}, platform={fp_platform}') co.headless(headless) co.set_argument('--no-sandbox') co.set_argument('--disable-dev-shm-usage') co.set_argument(f'--window-size={window_size}') co.set_argument('--disable-blink-features=AutomationControlled') co.set_argument(f"--fingerprint={fp_seed}") co.set_argument(f"--fingerprint-platform={fp_platform}") co.set_argument(f"--fingerprint-brand={fp_brand}") return co, tunnel # --------------------------------------------------------------------------- # 3. 浏览器内 JS Fetch 注入请求 # --------------------------------------------------------------------------- def perform_browser_fetch( page, method: str, url: str, headers: Optional[Dict[str, str]] = None, data: Optional[Any] = None, json_data: Optional[Any] = None, params: Optional[Dict[str, Any]] = None, timeout: int = 30 ) -> BrowserResponse: """ 在 DrissionPage 浏览器页面上下文内注入并执行原生 fetch 请求 """ if not page: raise BizLogicError("Browser page is not initialized") if params: query_str = urlencode(params) url += ('&' if '?' in url else '?') + query_str fetch_options: Dict[str, Any] = { "method": method.upper(), "headers": headers or {}, "credentials": "include" } if json_data is not None: fetch_options['body'] = json.dumps(json_data) fetch_options['headers']['Content-Type'] = 'application/json' elif data is not None: if isinstance(data, dict): fetch_options['body'] = urlencode(data) fetch_options['headers']['Content-Type'] = 'application/x-www-form-urlencoded' else: fetch_options['body'] = str(data) js_script = f""" const url = "{url}"; const options = {json.dumps(fetch_options)}; return fetch(url, options) .then(async response => {{ const text = await response.text(); const headers = {{}}; response.headers.forEach((value, key) => headers[key] = value); return {{ status: response.status, body: text, headers: headers, url: response.url }}; }}) .catch(error => {{ return {{ status: 0, body: error.toString(), headers: {{}}, url: url }}; }}); """ res_dict = page.run_js(js_script, timeout=timeout) return BrowserResponse(res_dict) # --------------------------------------------------------------------------- # 4. 辅助工具(截图、日期过滤、验证码求解、Cookie处理) # --------------------------------------------------------------------------- def save_screenshot( page, instance_id: str, name_prefix: str, save_dir: str = "data", full_page: bool = False, logger: Optional[Callable[[str], None]] = None ) -> Optional[str]: """保存页面截图至本地目录""" if not page: return None try: timestamp = int(time.time()) filename = f"{instance_id}_{name_prefix}_{timestamp}.jpg" save_path = os.path.join(save_dir, filename) os.makedirs(save_dir, exist_ok=True) page.get_screenshot(path=save_path, full_page=full_page) if logger: logger(f"Screenshot saved to {save_path}") return save_path except Exception as e: if logger: logger(f"Failed to save screenshot: {e}") return None def filter_dates_in_range( dates: List[str], start_str: Optional[str] = None, end_str: Optional[str] = None, shuffle: bool = True ) -> List[str]: """ 按起止日期范围筛选日期列表(格式: YYYY-MM-DD) """ if not start_str or not end_str: return list(dates) valid_dates = [] s_date = datetime.strptime(start_str[:10], "%Y-%m-%d") e_date = datetime.strptime(end_str[:10], "%Y-%m-%d") for date_str in dates: try: curr_date = datetime.strptime(date_str[:10], "%Y-%m-%d") if s_date <= curr_date <= e_date: valid_dates.append(date_str) except Exception: continue if shuffle: random.shuffle(valid_dates) return valid_dates def solve_capsolver_recaptcha( api_token: str, site_key: str, page_url: str, proxy: Optional[Any] = None, task_type: str = "ReCaptchaV2TaskProxyLess", action: Optional[str] = None ) -> str: """ 调用 Capsolver API 异步求解 ReCaptchaV2 """ if not api_token: raise NotFoundError("Api-token required for capsolver") import requests as req submit_url = "https://api.capsolver.com/createTask" task: Dict[str, Any] = { "type": task_type, "websiteURL": page_url, "websiteKey": site_key, } if action: task["pageAction"] = action if proxy and proxy.ip and not task_type.endswith("ProxyLess"): task["proxyType"] = proxy.proto task["proxyAddress"] = proxy.ip task["proxyPort"] = proxy.port if getattr(proxy, 'username', None): task["proxyLogin"] = proxy.username task["proxyPassword"] = proxy.password payload = {"clientKey": api_token, "task": task} r = req.post(submit_url, json=payload, timeout=20) if r.status_code != 200: raise BizLogicError(message="Failed to submit capsolver task") task_id = r.json().get("taskId") for _ in range(20): res = req.post("https://api.capsolver.com/getTaskResult", json={"clientKey": api_token, "taskId": task_id}, timeout=20) if res.status_code == 200: d = res.json() if d.get("status") == "ready": return d["solution"]["gRecaptchaResponse"] time.sleep(3) raise BizLogicError(message="Capsolver task timeout") def solve_capsolver_hcaptcha( api_token: str, site_key: str, page_url: str, proxy: Optional[Any] = None, task_type: str = "HCaptchaTaskProxyLess", logger: Optional[Callable[[str], None]] = None ) -> str: """ 调用 Capsolver API 异步求解 hCaptcha """ if not api_token: raise NotFoundError("Capsolver API key is required") def _log(msg: str): if logger: logger(msg) import requests as req submit_url = "https://api.capsolver.com/createTask" task: Dict[str, Any] = { "type": task_type, "websiteURL": page_url, "websiteKey": site_key, } if proxy and getattr(proxy, 'ip', None): task["proxyType"] = proxy.proto task["proxyAddress"] = proxy.ip task["proxyPort"] = proxy.port if getattr(proxy, 'username', None): task["proxyLogin"] = proxy.username task["proxyPassword"] = proxy.password payload = {"clientKey": api_token, "task": task} _log("Submitting hCaptcha task to Capsolver...") r = req.post(submit_url, json=payload, timeout=20) if r.status_code != 200: raise BizLogicError(message=f"Failed to create Capsolver task: {r.text}") res_json = r.json() if res_json.get("errorId", 0) != 0: raise BizLogicError(message=f"Failed to create Capsolver task: {r.text}") task_id = res_json.get("taskId") if not task_id: raise BizLogicError(message=f"No taskId returned from Capsolver: {r.text}") _log(f"Waiting for hCaptcha solution (TaskId: {task_id})...") start_time = time.time() while time.time() - start_time < 120: time.sleep(3) res = req.post( "https://api.capsolver.com/getTaskResult", json={"clientKey": api_token, "taskId": task_id}, timeout=20 ) if res.status_code == 200: result = res.json() status = result.get("status") if status == "ready": solution = result.get("solution", {}) token = solution.get("gRecaptchaResponse") or solution.get("token") or solution.get("response") _log("✅ hCaptcha solved successfully.") return str(token) if status == "failed": raise BizLogicError(message=f"hCaptcha solve failed: {res.text}") raise BizLogicError(message="Capsolver task timed out (120s)") def handle_cookie_consent( page, mouse=None, timeout: int = 3, logger: Optional[Callable[[str], None]] = None ) -> bool: """ 通用 Cookie Consent 弹窗识别与自动处理(支持 OneTrust、Osano 等常见弹窗) """ if not page: return False # 1. 优先尝试 OneTrust 策略 (VFS 等站点) try: onetrust_js = """ try { var acceptBtn = document.getElementById('onetrust-accept-btn-handler'); if (acceptBtn) { acceptBtn.click(); return 'clicked'; } var banner = document.getElementById('onetrust-banner-sdk'); if (banner) { banner.style.display = 'none'; banner.remove(); return 'removed'; } } catch(e) {} return 'none'; """ ot_res = page.run_js(onetrust_js) if ot_res in ('clicked', 'removed'): if logger: logger(f"Handled OneTrust cookie banner ({ot_res}).") return True except Exception: pass # 2. 尝试 Osano 及 ARIA 属性弹窗 (TLS 等站点) try: dialog = page.ele('@aria-label=Cookie Consent Banner', timeout=timeout) if dialog: target_btn = dialog.ele('tag:button@text():Save') or dialog.ele('tag:button@text():Accept') or dialog.ele('tag:button@text():Accept all') if target_btn: if mouse: mouse.human_click_ele(target_btn) else: target_btn.click() if logger: logger("Handled cookie consent dialog successfully.") return True except Exception as e: if logger: logger(f"Exception handling cookie dialog: {e}") return False def cleanup_browser_workspace( page=None, root_workspace: Optional[str] = None, tunnel=None, logger: Optional[Callable[[str], None]] = None ) -> None: """ 统一安全销毁浏览器实例、删除本地临时 workspace 目录并停止代理隧道 """ def _log(msg: str): if logger: logger(msg) errors = [] if root_workspace is not None: root_workspace = os.fspath(root_workspace) browser_closed = page is None process = None if page is not None: try: import psutil pid = page.process_id if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 and pid != os.getpid(): process = psutil.Process(pid) except psutil.NoSuchProcess: browser_closed = True except Exception: pass for attempt in range(2): try: page.quit(force=True, timeout=1) browser_closed = True break except Exception as exc: if process is not None and not process.is_running(): browser_closed = True break _log(f"Browser quit attempt {attempt + 1} failed: {exc}") if process is not None and process.is_running(): try: # Only the PID supplied by this page and its children are owned. # psutil.Process also detects PID reuse through creation time. children = process.children(recursive=True) owned = children + [process] for child in owned: try: child.terminate() except psutil.NoSuchProcess: pass _, alive = psutil.wait_procs(owned, timeout=0.5) for child in alive: child.kill() _, alive = psutil.wait_procs(alive, timeout=0.5) browser_closed = not alive except Exception as exc: browser_closed = False errors.append(f"Owned browser process shutdown failed: {exc}") if not browser_closed: errors.append("Browser did not confirm shutdown; retaining its handle") if tunnel is not None: try: tunnel.stop() except Exception as exc: errors.append(f"Tunnel shutdown failed: {exc}") if browser_closed and root_workspace and os.path.exists(root_workspace): for _ in range(3): try: shutil.rmtree(root_workspace) break except FileNotFoundError: break except OSError: time.sleep(0.1) if os.path.exists(root_workspace): errors.append(f"Browser workspace still exists: {root_workspace}") if errors: raise RuntimeError("; ".join(errors))