de_plugin.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555
  1. import time
  2. import json
  3. import random
  4. import re
  5. import os
  6. import uuid
  7. import shutil
  8. import base64
  9. import socket
  10. from datetime import datetime
  11. from typing import List, Dict, Optional, Any, Callable
  12. from urllib.parse import urljoin, urlparse, urlencode
  13. # DrissionPage 核心
  14. from DrissionPage import ChromiumPage, ChromiumOptions
  15. from vs_plg import IVSPlg
  16. from vs_types import VSPlgConfig, AppointmentType, VSQueryResult, VSBookResult, AvailabilityStatus, TimeSlot, DateAvailability, NotFoundError, PermissionDeniedError, RateLimiteddError, SessionExpiredOrInvalidError, BizLogicError, PluginStatus, utc_now_str
  17. from toolkit.vs_cloud_api import VSCloudApi
  18. from utils.cloudflare_bypass_for_scraping import CloudflareBypasser
  19. from toolkit.proxy_tunnel import ProxyTunnel
  20. from toolkit.ocr_engine import DddOcrEngine
  21. class BrowserResponse:
  22. def __init__(self, result_dict):
  23. result_dict = result_dict or {}
  24. self.status_code = result_dict.get('status', 0)
  25. self.text = result_dict.get('body', '')
  26. self.headers = result_dict.get('headers', {})
  27. self.url = result_dict.get('url', '')
  28. self._json = None
  29. def json(self):
  30. if self._json is None:
  31. if not self.text: return {}
  32. try: self._json = json.loads(self.text)
  33. except: self._json = {}
  34. return self._json
  35. def to_yyyymmdd(data_str: str, date_str_format: str, target_format: str="%Y-%m-%d"):
  36. dt = datetime.strptime(data_str, date_str_format)
  37. return dt.strftime("%Y-%m-%d")
  38. def get_alias_email(email: str, new_domain: str = "text.skin") -> str:
  39. if "@" not in email: raise ValueError(f"Invalid email: {email}")
  40. local_part, _ = email.rsplit("@", 1)
  41. return f"{local_part}@{new_domain}"
  42. class DePlugin(IVSPlg):
  43. """
  44. Germany (Visametric) 签证预约插件 (Browser + Tunnel Mode)
  45. """
  46. name = "de_plugin"
  47. version = "1.0.0"
  48. description = "德国Visametric签证预约插件 (支持Cloudflare穿透与OCR验证)"
  49. def __init__(self, group_id: str):
  50. self.group_id = group_id
  51. self.config: Optional[VSPlgConfig] = None
  52. self.free_config: Dict[str, Any] = {}
  53. self.logger = None
  54. # 浏览器实例
  55. self.page: Optional[ChromiumPage] = None
  56. # 资源隔离
  57. self.instance_id = uuid.uuid4().hex[:8]
  58. self.root_workspace = os.path.abspath(os.path.join("data/temp_browser_data", f"{self.group_id}.{self.instance_id}"))
  59. self.user_data_path = os.path.join(self.root_workspace, "user_data")
  60. if not os.path.exists(self.root_workspace):
  61. os.makedirs(self.root_workspace)
  62. self.tunnel = None # 代理隧道
  63. self.is_healthy = True
  64. self.session_create_time: float = 0
  65. # 字符识别引擎
  66. self.ocr_engine: Optional[DddOcrEngine] = None
  67. # 业务状态
  68. self.base_url = "https://ie-appointment.visametric.com"
  69. self.csrf_token = ""
  70. self.personal_info_val = ""
  71. self.email_val_control = ""
  72. def set_log(self, logger: Callable[[str], None]) -> None:
  73. self.logger = logger
  74. def _log(self, message):
  75. if self.logger:
  76. self.logger(f'[DePlugin] [{self.group_id}] {message}')
  77. else:
  78. print(f'[DePlugin] [{self.group_id}] {message}')
  79. def set_config(self, config: VSPlgConfig):
  80. self.config = config
  81. self.free_config = config.free_config or {}
  82. if self.free_config.get("base_url"):
  83. self.base_url = self.free_config["base_url"].rstrip('/')
  84. def keep_alive(self):
  85. pass
  86. def health_check(self) -> bool:
  87. if not self.is_healthy:
  88. return False
  89. if not self.page:
  90. return False
  91. try:
  92. if not self.page.run_js("return 1;"):
  93. return False
  94. except:
  95. return False
  96. if self.config.session_max_life > 0:
  97. if time.time() - self.session_create_time > self.config.session_max_life:
  98. self._log("Session expired.")
  99. return False
  100. return True
  101. def create_session(self):
  102. """
  103. 创建会话:启动浏览器 -> 代理隧道 -> 过盾 -> 提取 Captcha -> 本地识别 -> 提交 -> 获取 Context
  104. """
  105. self._log(f"Initializing Session (ID: {self.instance_id})...")
  106. self.ocr_engine = DddOcrEngine()
  107. co = ChromiumOptions()
  108. # 端口分配 (Docker 适配)
  109. def get_free_port():
  110. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
  111. s.bind(('', 0)); return s.getsockname()[1]
  112. co.set_local_port(get_free_port())
  113. # 路径与隔离
  114. co.set_user_data_path(self.user_data_path)
  115. chrome_path = os.getenv("CHROME_BIN")
  116. if chrome_path and os.path.exists(chrome_path):
  117. co.set_paths(browser_path=chrome_path)
  118. # 代理隧道
  119. if self.config.proxy and self.config.proxy.ip:
  120. p = self.config.proxy
  121. if p.username and p.password:
  122. self._log(f"Starting Tunnel for {p.ip}...")
  123. self.tunnel = ProxyTunnel(p.ip, p.port, p.username, p.password)
  124. local_proxy = self.tunnel.start()
  125. self._log(f"Tunnel started at {local_proxy}")
  126. co.set_argument(f'--proxy-server={local_proxy}')
  127. else:
  128. proxy_str = f"{p.proto}://{p.ip}:{p.port}"
  129. co.set_argument(f'--proxy-server={proxy_str}')
  130. else:
  131. self._log("[WARN] No proxy configured!")
  132. # Docker 核心参数
  133. co.headless(False)
  134. co.set_argument('--no-sandbox')
  135. co.set_argument('--disable-gpu')
  136. co.set_argument('--disable-dev-shm-usage')
  137. co.set_argument('--window-size=1920,1080')
  138. co.set_argument('--disable-blink-features=AutomationControlled')
  139. co.set_argument('--ignore-certificate-errors')
  140. try:
  141. self.page = ChromiumPage(co)
  142. # 1. 访问首页
  143. url_home = f"{self.base_url}/en"
  144. self._log(f"Navigating to {url_home}")
  145. self.page.get(url_home)
  146. # 2. Cloudflare 过盾
  147. cf = CloudflareBypasser(self.page, log=self.config.debug)
  148. if not cf.bypass(max_retry=15):
  149. if "access denied" in self.page.title.lower():
  150. raise BizLogicError("Cloudflare Access Denied")
  151. raise BizLogicError("Cloudflare bypass timeout")
  152. # 3. 提取 CSRF 和 验证码
  153. # 等待页面加载
  154. meta_ele = self.page.ele('xpath://meta[@name="csrf-token"]', timeout=30)
  155. if not meta_ele:
  156. # 截图调试,看看是不是还在 Cloudflare 或者加载失败
  157. self.page.get_screenshot(path='csrf_not_found.jpg')
  158. raise NotFoundError("CSRF Token meta tag not found (Page load failed?)")
  159. self.csrf_token = meta_ele.attr('content')
  160. # 提取验证码图片 (Visametric Base64)
  161. html = self.page.html
  162. match = re.search(r'"data:image/png;base64,"\s*\+\s*"(.*?)"', html)
  163. if not match:
  164. # 尝试直接找 img
  165. try:
  166. img_ele = self.page.ele('xpath://img[contains(@src, "data:image")]')
  167. if img_ele:
  168. b64_src = img_ele.attr('src')
  169. captcha_b64 = b64_src.split(',')[1]
  170. else:
  171. raise NotFoundError("Captcha image not found")
  172. except:
  173. raise NotFoundError("Captcha image not found (Regex failed)")
  174. else:
  175. captcha_b64 = match.group(1)
  176. image_bytes = base64.b64decode(captcha_b64)
  177. # 4. 识别验证码 (本地 OCR 服务)
  178. captcha_code = self.ocr_engine.inference_captcha(image_bytes)
  179. # 5. 提交验证码 (获取 PersonalInfo)
  180. self._submit_captcha(captcha_code)
  181. self.session_create_time = time.time()
  182. self._log("Session created successfully.")
  183. except Exception as e:
  184. self._log(f"Session Create Failed: {e}")
  185. self.cleanup()
  186. raise e
  187. def _submit_captcha(self, code):
  188. """
  189. 提交验证码,获取 personalinfo 和 emailValControl
  190. """
  191. url = f"{self.base_url}/en/appointment-form"
  192. payload = {
  193. '_token': self.csrf_token,
  194. 'cpJvnsControl': '',
  195. 'mailConfirmCode': code
  196. }
  197. # 使用 Fetch 提交 (Form-UrlEncoded)
  198. resp = self._perform_request('POST', url, data=payload, headers={
  199. 'X-Requested-With': 'XMLHttpRequest'
  200. })
  201. # 解析返回的 HTML 片段
  202. html = resp.text
  203. # 提取 personalinfo
  204. match_pi = re.search(r"personalinfo:\s*'([^']*)'", html)
  205. if match_pi: self.personal_info_val = match_pi.group(1)
  206. # 提取 emailValControl
  207. match_ev = re.search(r"emailValControl:\s*'([^']*)'", html)
  208. if match_ev: self.email_val_control = match_ev.group(1)
  209. if not self.personal_info_val:
  210. raise NotFoundError(message="Personalinfo not found in captcha response")
  211. # 更新 CSRF (如果返回了新的)
  212. m = re.search(r'name="csrf-token" content="([^"]+)"', html)
  213. if m: self.csrf_token = m.group(1)
  214. def query(self, apt_type: AppointmentType) -> VSQueryResult:
  215. res = VSQueryResult()
  216. res.success = False
  217. consular_id = self.free_config.get("consularid", "1")
  218. url = f"{self.base_url}/en/getdate"
  219. payload = {
  220. "consularid": consular_id,
  221. "exitid": "1",
  222. "servicetypeid": "1",
  223. "calendarType": "2",
  224. "totalperson": "1"
  225. }
  226. headers = {
  227. 'X-CSRF-TOKEN': self.csrf_token,
  228. 'X-Requested-With': 'XMLHttpRequest'
  229. }
  230. try:
  231. resp = self._perform_request('POST', url, data=payload, headers=headers, retry_count=1)
  232. except Exception as e:
  233. self._log(f"Query Error: {e}")
  234. raise e
  235. j = resp.json()
  236. dates = j.get("getDateEnable", [])
  237. if dates:
  238. res.success = True
  239. res.availability_status = AvailabilityStatus.Available
  240. earliest_date = dates[0]
  241. earliest_dt = datetime.strptime(earliest_date, "%d-%m-%Y")
  242. res.earliest_date = earliest_dt
  243. res.availability = [
  244. DateAvailability(date=datetime.strptime(d, "%d-%m-%Y"), times=[])
  245. for d in dates
  246. ]
  247. else:
  248. res.availability_status = AvailabilityStatus.NoneAvailable
  249. return res
  250. def book(self, slot_info: VSQueryResult, user_inputs: Dict) -> VSBookResult:
  251. res = VSBookResult()
  252. available_dates = [da.date for da in slot_info.availability]
  253. exp_start = user_inputs.get('expected_start_date', '')
  254. exp_end = user_inputs.get('expected_end_date', '')
  255. available_dates_str = [
  256. da.date.strftime("%Y-%m-%d")
  257. for da in slot_info.availability
  258. ]
  259. valid_dates = self._filter_dates(available_dates, exp_start, exp_end)
  260. if not valid_dates:
  261. raise NotFoundError("No dates match constraints")
  262. target_date = random.choice(valid_dates)
  263. self._log(f"Selected date: {target_date}")
  264. # 1. 获取时间 Slot
  265. time_slot = self._get_slot_time(target_date)
  266. # 2. 发送邮件流程
  267. alias_email = get_alias_email(user_inputs.get("email"), new_domain='text.skin')
  268. self._send_email_step1(alias_email)
  269. self._send_email_step2("0")
  270. # 3. 读取 OTP
  271. otp_code = self._read_otp_email(alias_email)
  272. # 4. 提交确认
  273. book_res_html = self._confirm_appointment(target_date, time_slot, user_inputs, otp_code, alias_email)
  274. if "complete all required fields" in book_res_html.lower():
  275. raise BizLogicError("Incomplete fields response")
  276. match = re.search(r'https:\/\/checkout\.stripe\.com\/c\/pay\/[^\s"]+', book_res_html)
  277. res.success = True
  278. res.fee_amount = 3000
  279. res.fee_currency = 'EUR'
  280. res.book_date = target_date
  281. res.book_time = time_slot['time']
  282. if match:
  283. res.payment_link = match.group(0)
  284. self._log(f"Payment Link: {res.payment_link}")
  285. return res
  286. # ---------------------------------------------------------
  287. # 辅助方法
  288. # ---------------------------------------------------------
  289. def _perform_request(self, method, url, headers=None, data=None, json_data=None, params=None, retry_count=0):
  290. if not self.page:
  291. raise BizLogicError("Browser not init")
  292. req_url = url
  293. if params:
  294. sep = '&' if '?' in req_url else '?'
  295. req_url += sep + urlencode(params)
  296. fetch_opts = { "method": method.upper(), "headers": headers or {}, "credentials": "include" }
  297. if json_data:
  298. fetch_opts['body'] = json.dumps(json_data)
  299. fetch_opts['headers']['Content-Type'] = 'application/json'
  300. elif data:
  301. if isinstance(data, dict):
  302. fetch_opts['body'] = urlencode(data)
  303. fetch_opts['headers']['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
  304. else:
  305. fetch_opts['body'] = data
  306. js = f"""
  307. return fetch("{req_url}", {json.dumps(fetch_opts)})
  308. .then(async r => {{
  309. const h = {{}}; r.headers.forEach((v, k) => h[k] = v);
  310. return {{ status: r.status, body: await r.text(), headers: h, url: r.url }};
  311. }}).catch(e => {{ return {{ status: 0, body: e.toString() }}; }});
  312. """
  313. resp = BrowserResponse(self.page.run_js(js, timeout=60))
  314. if resp.status_code == 200:
  315. return resp
  316. elif resp.status_code == 403:
  317. if "Just a moment" in resp.text and retry_count < 2:
  318. self._log("Cloudflare 403. Refreshing...")
  319. if self._refresh_firewall_session():
  320. return self._perform_request(method, url, headers, data, json_data, params, retry_count+1)
  321. raise PermissionDeniedError(f"HTTP 403: {resp.text[:100]}")
  322. elif resp.status_code == 429:
  323. self.is_healthy = False
  324. raise RateLimiteddError()
  325. elif resp.status_code in [401, 419]:
  326. self.is_healthy = False
  327. raise SessionExpiredOrInvalidError()
  328. else:
  329. raise BizLogicError(f"HTTP {resp.status_code}: {resp.text[:100]}")
  330. def _refresh_firewall_session(self):
  331. try:
  332. self.page.refresh()
  333. cf = CloudflareBypasser(self.page, log=self.config.debug)
  334. return cf.bypass(max_retry=10)
  335. except: return False
  336. def _get_slot_time(self, date) -> Dict:
  337. url = f"{self.base_url}/en/senddate"
  338. dt_m = datetime.strptime(date, "%Y-%m-%d")
  339. converted_date = dt_m.strftime("%d-%m-%Y")
  340. payload = {
  341. "fulldate": converted_date,
  342. "totalperson": "1",
  343. "set_new_consular_id": self.free_config.get("consularid", "1"),
  344. "set_new_exit_office_id": "1",
  345. "calendarType": "2",
  346. "set_new_service_type_id": "1",
  347. "personalinfo": self.personal_info_val
  348. }
  349. headers = {'X-CSRF-TOKEN': self.csrf_token, 'X-Requested-With': 'XMLHttpRequest'}
  350. resp = self._perform_request('POST', url, data=payload, headers=headers)
  351. # 使用 Regex 提取 Slot
  352. times = []
  353. # pattern: data-id="123" ... <i>09:00</i>
  354. for m in re.finditer(r'data-id="([^"]+)"[^>]*data-all="([^"]+)"[^>]*>.*?<i>(.*?)</i>', resp.text, re.DOTALL):
  355. times.append({'data_id': m.group(1), 'data_all': m.group(2), 'time': m.group(3).strip()})
  356. if not times: raise NotFoundError("No time slots")
  357. return random.choice(times)
  358. def _send_email_step1(self, email):
  359. url = f"{self.base_url}/en/jky45fgd"
  360. payload = { "emailCheck": email, "personalinfo": self.personal_info_val }
  361. headers = {'X-CSRF-TOKEN': self.csrf_token, 'X-Requested-With': 'XMLHttpRequest'}
  362. self._perform_request('POST', url, data=payload, headers=headers)
  363. def _send_email_step2(self, code_val):
  364. url = f"{self.base_url}/en/confirmCodeSendMail"
  365. payload = { "confirmCode": code_val, "emailValControl": self.email_val_control }
  366. headers = {'X-CSRF-TOKEN': self.csrf_token, 'X-Requested-With': 'XMLHttpRequest'}
  367. self._perform_request('POST', url, data=payload, headers=headers)
  368. def _read_otp_email(self, recipient) -> str:
  369. master_email = "hujiarui8@gmail.com"
  370. sender = 'Visametric - verify at visametric.com'
  371. now_utc = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
  372. for i in range(12):
  373. c = VSCloudApi.Instance().fetch_mail_content(master_email, sender, recipient, 'Verification Code', 'Verification code', now_utc, 300)
  374. if c:
  375. m = re.search(r'\b\d{6}\b', c)
  376. if m: return m.group(0)
  377. time.sleep(5)
  378. raise NotFoundError("OTP timeout")
  379. def _confirm_appointment(self, date, slot_data, user_inputs, otp, alias_email):
  380. url = f"{self.base_url}/en/personal/appointment/create"
  381. def _get_dob(d):
  382. try: return datetime.strptime(d[:10], "%Y-%m-%d")
  383. except: return datetime.now()
  384. dob = _get_dob(user_inputs.get('birthday', ''))
  385. payload = {
  386. "_token": self.csrf_token,
  387. "country": str(self.free_config.get("consularid", "1")),
  388. "visitingcountry": str(self.free_config.get("consularid", "1")),
  389. "city": "6",
  390. "office": "1",
  391. "officetype": "1",
  392. "totalPerson": "1",
  393. "name1": user_inputs.get('first_name', '').upper(),
  394. "surname1": user_inputs.get('last_name', '').upper(),
  395. "nationality1": "2",
  396. "birthday1": str(dob.day),
  397. "birthmonth1": str(dob.month),
  398. "birthyear1": str(dob.year),
  399. "passport1": user_inputs.get('passport_no'),
  400. "passportExpirationDate1": datetime.strptime(user_inputs.get('passport_expiry_date', '')[:10], "%Y-%m-%d").strftime("%d-%m-%Y"),
  401. "email1": alias_email,
  402. "phone1": user_inputs.get('phone_no'),
  403. "alternativephone1": "",
  404. "mailConfirmCode": otp,
  405. "ctval": slot_data['data_id'],
  406. "qtallvert": slot_data['data_all'],
  407. "oldofficetype": "1",
  408. "oldtotalperson": "1",
  409. "rePaymentControl": "0",
  410. "view_set_app_country": "Schengen - Tourism/Family&Friend Visit/Transit Visa/Other Purposes",
  411. "view_set_app_office": "Dublin",
  412. "view_set_app_service_type": "NORMAL",
  413. "cargoactive": "0",
  414. "setnewcalendarstatus": "2",
  415. "availableDaycontrol": "0",
  416. "travelStartDate": datetime.strptime(user_inputs.get('travel_date', '')[:10], "%Y-%m-%d").strftime("%d-%m-%Y"),
  417. "personalapproveTerms": "1"
  418. }
  419. # 补全空字段 (Person 2-4)
  420. for i in range(2, 5):
  421. payload.update({
  422. f"name{i}": "", f"surname{i}": "", f"nationality{i}": "0", f"birthday{i}": "0", f"birthmonth{i}": "0", f"birthyear{i}": "0", f"passport{i}": "", f"passportExpirationDate{i}": "", f"email{i}": alias_email, f"phone{i}": user_inputs.get('phone_no'), f"alternativephone{i}": ""
  423. })
  424. headers = {'X-Requested-With': 'XMLHttpRequest'}
  425. return self._perform_request('POST', url, data=payload, headers=headers).text
  426. def _filter_dates(self, dates, start, end):
  427. if not start or not end: return dates
  428. valid = []
  429. s = datetime.strptime(start[:10], "%Y-%m-%d")
  430. e = datetime.strptime(end[:10], "%Y-%m-%d")
  431. for d in dates:
  432. c = datetime.strptime(d, "%Y-%m-%d")
  433. if s <= c <= e: valid.append(d)
  434. random.shuffle(valid)
  435. return valid
  436. def cleanup(self):
  437. if self.page:
  438. try: self.page.quit()
  439. except: pass
  440. self.page = None
  441. if os.path.exists(self.root_workspace):
  442. for _ in range(3):
  443. try: time.sleep(0.2); shutil.rmtree(self.root_workspace, ignore_errors=True); break
  444. except: time.sleep(0.5)
  445. if self.tunnel:
  446. try: self.tunnel.stop()
  447. except: pass
  448. self.tunnel = None
  449. def get_plugin_status(self) -> PluginStatus:
  450. account_name = self.config.account.username if self.config and self.config.account else ""
  451. proxy_info = f"{self.config.proxy.ip}:{self.config.proxy.port}" if self.config and self.config.proxy and self.config.proxy.ip else ""
  452. current_url = ""
  453. if self.page:
  454. try:
  455. current_url = self.page.url
  456. except Exception:
  457. pass
  458. state = "running" if getattr(self, "is_healthy", True) else "unhealthy"
  459. return PluginStatus(
  460. instance_id=getattr(self, "instance_id", self.group_id),
  461. is_healthy=getattr(self, "is_healthy", True),
  462. state=state,
  463. current_url=current_url,
  464. account=account_name,
  465. proxy=proxy_info,
  466. session_age_sec=0,
  467. last_active_at=utc_now_str(),
  468. )
  469. def __del__(self):
  470. self.cleanup()