| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529 |
- """
- plugin_utils.py - 签证插件通用基础工具模块
- 提供浏览器响应封装、Options/隧道初始化、JS Fetch 注入、截图、日期筛选、验证码求解及 Cookie 弹窗处理等通用函数
- """
- import os
- import json
- import time
- import socket
- import random
- import shutil
- from datetime import datetime
- from typing import List, Dict, Optional, Any, Callable, Tuple
- from urllib.parse import urlencode
- import configure
- from core_types import BizLogicError, NotFoundError
- from toolkit.mihomo_tunnel import MihomoTunnel
- from utils.fingerprint_utils import FingerprintGenerator
- from DrissionPage import ChromiumOptions
- # ---------------------------------------------------------------------------
- # 1. 响应封装类
- # ---------------------------------------------------------------------------
- class BrowserResponse:
- """封装在浏览器上下文内执行 JS Fetch 返回的响应对象"""
- def __init__(self, result_dict: Optional[Dict[str, Any]] = None):
- result_dict = result_dict or {}
- self.status_code = result_dict.get('status', 0)
- self.text = result_dict.get('body', '')
- self.headers = result_dict.get('headers', {})
- self.url = result_dict.get('url', '')
- self._json = None
- def json(self) -> Dict[str, Any]:
- if self._json is None:
- if not self.text:
- return {}
- try:
- self._json = json.loads(self.text)
- except Exception:
- self._json = {}
- return self._json
- @property
- def content(self) -> bytes:
- return self.text.encode('utf-8')
- # ---------------------------------------------------------------------------
- # 2. 网络端口与浏览器启动选项
- # ---------------------------------------------------------------------------
- def get_free_port() -> int:
- """获取本地随机空闲 TCP 端口"""
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
- s.bind(('', 0))
- return s.getsockname()[1]
- def init_chromium_options(
- user_data_path: str,
- username_seed: Optional[str] = None,
- proxy: Optional[Any] = None,
- headless: bool = False,
- window_size: str = "1920,1080",
- debug_port: Optional[int] = None,
- logger: Optional[Callable[[str], None]] = None
- ) -> Tuple[ChromiumOptions, Optional[MihomoTunnel]]:
- """
- 配置并返回 ChromiumOptions 以及(若启用了中转)创建好的 MihomoTunnel 实例
-
- :param user_data_path: 浏览器用户数据存储目录
- :param username_seed: 用于生成确定性指纹的账号特征种子
- :param proxy: 代理配置对象(含 ip, port, proto, username, password)
- :param headless: 是否使用无头模式
- :param window_size: 初始窗口尺寸 (宽,高)
- :param debug_port: 本地调试端口,不传则自动获取
- :param logger: 日志回调函数
- :return: (co, tunnel)
- """
- def _log(msg: str):
- if logger:
- logger(msg)
- co = ChromiumOptions()
- port = debug_port if debug_port is not None else get_free_port()
- co.set_local_port(port)
- co.set_user_data_path(user_data_path)
-
- chrome_path = configure.CHROME_PATH or os.getenv("CHROME_BIN")
- if chrome_path and os.path.exists(chrome_path):
- co.set_paths(browser_path=chrome_path)
- tunnel = None
- if proxy and getattr(proxy, 'ip', None):
- if getattr(proxy, 'username', None) and getattr(proxy, 'password', None):
- _log(f"Starting Proxy Tunnel for {proxy.ip}:{proxy.port}...")
- exit_node = {
- "name": "ExitNode",
- "type": proxy.proto,
- "server": proxy.ip,
- "port": proxy.port,
- "username": proxy.username,
- "password": proxy.password
- }
- relay_node = None
- if configure.MIHOMO_RELAY_NODES:
- relay_node = random.choice(configure.MIHOMO_RELAY_NODES)
- mihomo_path = configure.MIHOMO_BIN_PATH or os.getenv("MIHOMO_BIN")
- if not mihomo_path:
- raise BizLogicError(message='Mihomo path is null, You need set mihomo bin path in configure or os env')
- tunnel = MihomoTunnel(mihomo_path, exit_node=exit_node, relay_node=relay_node)
- local_proxy = tunnel.start()
- _log(f"Tunnel started at {local_proxy}")
- co.set_argument(f'--proxy-server={local_proxy}')
- else:
- proxy_str = f"{proxy.proto}://{proxy.ip}:{proxy.port}"
- co.set_argument(f'--proxy-server={proxy_str}')
- else:
- _log("[WARN] No proxy configured!")
- seed = username_seed or "default"
- specific_fp = FingerprintGenerator().generate(seed)
- fp_seed = specific_fp.get("seed")
- fp_platform = specific_fp.get("platform")
- fp_brand = specific_fp.get("brand")
- _log(f'Browser fingerprint seed={fp_seed}, platform={fp_platform}')
- co.headless(headless)
- co.set_argument('--no-sandbox')
- co.set_argument('--disable-dev-shm-usage')
- co.set_argument(f'--window-size={window_size}')
- co.set_argument('--disable-blink-features=AutomationControlled')
- co.set_argument(f"--fingerprint={fp_seed}")
- co.set_argument(f"--fingerprint-platform={fp_platform}")
- co.set_argument(f"--fingerprint-brand={fp_brand}")
- return co, tunnel
- # ---------------------------------------------------------------------------
- # 3. 浏览器内 JS Fetch 注入请求
- # ---------------------------------------------------------------------------
- def perform_browser_fetch(
- page,
- method: str,
- url: str,
- headers: Optional[Dict[str, str]] = None,
- data: Optional[Any] = None,
- json_data: Optional[Any] = None,
- params: Optional[Dict[str, Any]] = None,
- timeout: int = 30
- ) -> BrowserResponse:
- """
- 在 DrissionPage 浏览器页面上下文内注入并执行原生 fetch 请求
- """
- if not page:
- raise BizLogicError("Browser page is not initialized")
- if params:
- query_str = urlencode(params)
- url += ('&' if '?' in url else '?') + query_str
- fetch_options: Dict[str, Any] = {
- "method": method.upper(),
- "headers": headers or {},
- "credentials": "include"
- }
- if json_data is not None:
- fetch_options['body'] = json.dumps(json_data)
- fetch_options['headers']['Content-Type'] = 'application/json'
- elif data is not None:
- if isinstance(data, dict):
- fetch_options['body'] = urlencode(data)
- fetch_options['headers']['Content-Type'] = 'application/x-www-form-urlencoded'
- else:
- fetch_options['body'] = str(data)
- js_script = f"""
- const url = "{url}";
- const options = {json.dumps(fetch_options)};
-
- return fetch(url, options)
- .then(async response => {{
- const text = await response.text();
- const headers = {{}};
- response.headers.forEach((value, key) => headers[key] = value);
- return {{
- status: response.status,
- body: text,
- headers: headers,
- url: response.url
- }};
- }})
- .catch(error => {{
- return {{
- status: 0,
- body: error.toString(),
- headers: {{}},
- url: url
- }};
- }});
- """
- res_dict = page.run_js(js_script, timeout=timeout)
- return BrowserResponse(res_dict)
- # ---------------------------------------------------------------------------
- # 4. 辅助工具(截图、日期过滤、验证码求解、Cookie处理)
- # ---------------------------------------------------------------------------
- def save_screenshot(
- page,
- instance_id: str,
- name_prefix: str,
- save_dir: str = "data",
- full_page: bool = False,
- logger: Optional[Callable[[str], None]] = None
- ) -> Optional[str]:
- """保存页面截图至本地目录"""
- if not page:
- return None
- try:
- timestamp = int(time.time())
- filename = f"{instance_id}_{name_prefix}_{timestamp}.jpg"
- save_path = os.path.join(save_dir, filename)
- os.makedirs(save_dir, exist_ok=True)
- page.get_screenshot(path=save_path, full_page=full_page)
- if logger:
- logger(f"Screenshot saved to {save_path}")
- return save_path
- except Exception as e:
- if logger:
- logger(f"Failed to save screenshot: {e}")
- return None
- def filter_dates_in_range(
- dates: List[str],
- start_str: Optional[str] = None,
- end_str: Optional[str] = None,
- shuffle: bool = True
- ) -> List[str]:
- """
- 按起止日期范围筛选日期列表(格式: YYYY-MM-DD)
- """
- if not start_str or not end_str:
- return list(dates)
-
- valid_dates = []
- s_date = datetime.strptime(start_str[:10], "%Y-%m-%d")
- e_date = datetime.strptime(end_str[:10], "%Y-%m-%d")
- for date_str in dates:
- try:
- curr_date = datetime.strptime(date_str[:10], "%Y-%m-%d")
- if s_date <= curr_date <= e_date:
- valid_dates.append(date_str)
- except Exception:
- continue
- if shuffle:
- random.shuffle(valid_dates)
- return valid_dates
- def solve_capsolver_recaptcha(
- api_token: str,
- site_key: str,
- page_url: str,
- proxy: Optional[Any] = None,
- task_type: str = "ReCaptchaV2TaskProxyLess",
- action: Optional[str] = None
- ) -> str:
- """
- 调用 Capsolver API 异步求解 ReCaptchaV2
- """
- if not api_token:
- raise NotFoundError("Api-token required for capsolver")
- import requests as req
- submit_url = "https://api.capsolver.com/createTask"
- task: Dict[str, Any] = {
- "type": task_type,
- "websiteURL": page_url,
- "websiteKey": site_key,
- }
- if action:
- task["pageAction"] = action
- if proxy and proxy.ip and not task_type.endswith("ProxyLess"):
- task["proxyType"] = proxy.proto
- task["proxyAddress"] = proxy.ip
- task["proxyPort"] = proxy.port
- if getattr(proxy, 'username', None):
- task["proxyLogin"] = proxy.username
- task["proxyPassword"] = proxy.password
- payload = {"clientKey": api_token, "task": task}
- r = req.post(submit_url, json=payload, timeout=20)
- if r.status_code != 200:
- raise BizLogicError(message="Failed to submit capsolver task")
- task_id = r.json().get("taskId")
- for _ in range(20):
- res = req.post("https://api.capsolver.com/getTaskResult", json={"clientKey": api_token, "taskId": task_id}, timeout=20)
- if res.status_code == 200:
- d = res.json()
- if d.get("status") == "ready":
- return d["solution"]["gRecaptchaResponse"]
- time.sleep(3)
- raise BizLogicError(message="Capsolver task timeout")
- def solve_capsolver_hcaptcha(
- api_token: str,
- site_key: str,
- page_url: str,
- proxy: Optional[Any] = None,
- task_type: str = "HCaptchaTaskProxyLess",
- logger: Optional[Callable[[str], None]] = None
- ) -> str:
- """
- 调用 Capsolver API 异步求解 hCaptcha
- """
- if not api_token:
- raise NotFoundError("Capsolver API key is required")
- def _log(msg: str):
- if logger:
- logger(msg)
- import requests as req
- submit_url = "https://api.capsolver.com/createTask"
- task: Dict[str, Any] = {
- "type": task_type,
- "websiteURL": page_url,
- "websiteKey": site_key,
- }
- if proxy and getattr(proxy, 'ip', None):
- task["proxyType"] = proxy.proto
- task["proxyAddress"] = proxy.ip
- task["proxyPort"] = proxy.port
- if getattr(proxy, 'username', None):
- task["proxyLogin"] = proxy.username
- task["proxyPassword"] = proxy.password
- payload = {"clientKey": api_token, "task": task}
- _log("Submitting hCaptcha task to Capsolver...")
- r = req.post(submit_url, json=payload, timeout=20)
- if r.status_code != 200:
- raise BizLogicError(message=f"Failed to create Capsolver task: {r.text}")
- res_json = r.json()
- if res_json.get("errorId", 0) != 0:
- raise BizLogicError(message=f"Failed to create Capsolver task: {r.text}")
- task_id = res_json.get("taskId")
- if not task_id:
- raise BizLogicError(message=f"No taskId returned from Capsolver: {r.text}")
- _log(f"Waiting for hCaptcha solution (TaskId: {task_id})...")
- start_time = time.time()
- while time.time() - start_time < 120:
- time.sleep(3)
- res = req.post(
- "https://api.capsolver.com/getTaskResult",
- json={"clientKey": api_token, "taskId": task_id},
- timeout=20
- )
- if res.status_code == 200:
- result = res.json()
- status = result.get("status")
- if status == "ready":
- solution = result.get("solution", {})
- token = solution.get("gRecaptchaResponse") or solution.get("token") or solution.get("response")
- _log("✅ hCaptcha solved successfully.")
- return str(token)
- if status == "failed":
- raise BizLogicError(message=f"hCaptcha solve failed: {res.text}")
- raise BizLogicError(message="Capsolver task timed out (120s)")
- def handle_cookie_consent(
- page,
- mouse=None,
- timeout: int = 3,
- logger: Optional[Callable[[str], None]] = None
- ) -> bool:
- """
- 通用 Cookie Consent 弹窗识别与自动处理(支持 OneTrust、Osano 等常见弹窗)
- """
- if not page:
- return False
-
- # 1. 优先尝试 OneTrust 策略 (VFS 等站点)
- try:
- onetrust_js = """
- try {
- var acceptBtn = document.getElementById('onetrust-accept-btn-handler');
- if (acceptBtn) {
- acceptBtn.click();
- return 'clicked';
- }
- var banner = document.getElementById('onetrust-banner-sdk');
- if (banner) {
- banner.style.display = 'none';
- banner.remove();
- return 'removed';
- }
- } catch(e) {}
- return 'none';
- """
- ot_res = page.run_js(onetrust_js)
- if ot_res in ('clicked', 'removed'):
- if logger:
- logger(f"Handled OneTrust cookie banner ({ot_res}).")
- return True
- except Exception:
- pass
- # 2. 尝试 Osano 及 ARIA 属性弹窗 (TLS 等站点)
- try:
- dialog = page.ele('@aria-label=Cookie Consent Banner', timeout=timeout)
- if dialog:
- target_btn = dialog.ele('tag:button@text():Save') or dialog.ele('tag:button@text():Accept') or dialog.ele('tag:button@text():Accept all')
- if target_btn:
- if mouse:
- mouse.human_click_ele(target_btn)
- else:
- target_btn.click()
- if logger:
- logger("Handled cookie consent dialog successfully.")
- return True
- except Exception as e:
- if logger:
- logger(f"Exception handling cookie dialog: {e}")
- return False
- def cleanup_browser_workspace(
- page=None,
- root_workspace: Optional[str] = None,
- tunnel=None,
- logger: Optional[Callable[[str], None]] = None
- ) -> None:
- """
- 统一安全销毁浏览器实例、删除本地临时 workspace 目录并停止代理隧道
- """
- def _log(msg: str):
- if logger:
- logger(msg)
- errors = []
- if root_workspace is not None:
- root_workspace = os.fspath(root_workspace)
- browser_closed = page is None
- process = None
- if page is not None:
- try:
- import psutil
- pid = page.process_id
- if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 and pid != os.getpid():
- process = psutil.Process(pid)
- except psutil.NoSuchProcess:
- browser_closed = True
- except Exception:
- pass
- for attempt in range(2):
- try:
- page.quit(force=True, timeout=1)
- browser_closed = True
- break
- except Exception as exc:
- if process is not None and not process.is_running():
- browser_closed = True
- break
- _log(f"Browser quit attempt {attempt + 1} failed: {exc}")
- if process is not None and process.is_running():
- try:
- # Only the PID supplied by this page and its children are owned.
- # psutil.Process also detects PID reuse through creation time.
- children = process.children(recursive=True)
- owned = children + [process]
- for child in owned:
- try:
- child.terminate()
- except psutil.NoSuchProcess:
- pass
- _, alive = psutil.wait_procs(owned, timeout=0.5)
- for child in alive:
- child.kill()
- _, alive = psutil.wait_procs(alive, timeout=0.5)
- browser_closed = not alive
- except Exception as exc:
- browser_closed = False
- errors.append(f"Owned browser process shutdown failed: {exc}")
- if not browser_closed:
- errors.append("Browser did not confirm shutdown; retaining its handle")
- if tunnel is not None:
- try:
- tunnel.stop()
- except Exception as exc:
- errors.append(f"Tunnel shutdown failed: {exc}")
- if browser_closed and root_workspace and os.path.exists(root_workspace):
- for _ in range(3):
- try:
- shutil.rmtree(root_workspace)
- break
- except FileNotFoundError:
- break
- except OSError:
- time.sleep(0.1)
- if os.path.exists(root_workspace):
- errors.append(f"Browser workspace still exists: {root_workspace}")
- if errors:
- raise RuntimeError("; ".join(errors))
|