cloudflare_bypass_for_scraping.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. """
  2. Cloudflare 挑战自动化穿透工具模块。
  3. 针对 Cloudflare Turnstile / 5秒盾(Managed Challenge)提供高效、稳定的穿透能力。
  4. 核心设计理念:
  5. 1. 快速精准定位:定向穿透隐藏 turnstile input 对应的 shadow_root 宿主容器及嵌套 iframe,
  6. 无需遍历全局 DOM,执行耗时 < 0.2s。
  7. 2. 异常自愈回退:提供可控深度的 DFS 备用探测策略。
  8. 3. 等待室自适应排队:支持 Waiting Room 自动轮询处理。
  9. 4. 代码风格标准:严格遵循 Pythonic 规范,杜绝使用 hasattr,保证属性访问安全与可维护性。
  10. """
  11. import time
  12. from typing import Optional, Callable
  13. from DrissionPage import ChromiumPage
  14. from DrissionPage._elements.chromium_element import ChromiumElement
  15. class CloudflareBypasser:
  16. """
  17. Cloudflare 页面挑战识别与穿透处理器
  18. """
  19. def __init__(
  20. self,
  21. driver: ChromiumPage,
  22. log: bool = False,
  23. logger: Optional[Callable[[str], None]] = None
  24. ) -> None:
  25. self.driver = driver
  26. self.log = log
  27. self.logger = logger
  28. def log_message(self, message: str) -> None:
  29. """统一日志输出辅助方法"""
  30. if not self.log:
  31. return
  32. formatted = f"[CF-Bypass] {message}"
  33. if self.logger:
  34. if callable(self.logger):
  35. try:
  36. self.logger(formatted)
  37. except TypeError:
  38. self.logger(message)
  39. else:
  40. log_fn = getattr(self.logger, "info", print)
  41. log_fn(formatted)
  42. else:
  43. print(formatted)
  44. def _extract_turnstile_button_from_host(self, host_ele: Optional[ChromiumElement]) -> Optional[ChromiumElement]:
  45. """
  46. 从挂载 Shadow Root 的宿主容器元素中提取 Turnstile 复选框按钮。
  47. DOM 层级链路:
  48. Host Element (div)
  49. └─ shadow_root
  50. └─ child (iframe)
  51. └─ body
  52. └─ shadow_root
  53. └─ input[type=checkbox]
  54. """
  55. if not host_ele:
  56. return None
  57. try:
  58. host_sr = getattr(host_ele, "shadow_root", None)
  59. if not host_sr:
  60. return None
  61. iframe = host_sr.child()
  62. if not iframe:
  63. return None
  64. body = iframe("tag:body")
  65. if not body:
  66. return None
  67. body_sr = getattr(body, "shadow_root", None)
  68. if not body_sr:
  69. return None
  70. return body_sr("tag:input")
  71. except Exception as e:
  72. self.log_message(f"Error extracting button from host element: {e}")
  73. return None
  74. def locate_cf_button(self, dfs: bool = False) -> Optional[ChromiumElement]:
  75. """
  76. 定位 Cloudflare 验证复选框按钮。
  77. 策略 1(主路径,毫秒级):扫描主文档中的 'cf-turnstile-response' 隐藏 input,
  78. 其父级容器或同级兄弟 div 即为真实 shadow_root 宿主,直接提取目标按钮。
  79. 策略 2(备用路径):若主路径未命中且显式开启 dfs=True,则从 body 递归搜索。
  80. """
  81. try:
  82. # --- 策略 1: 基于隐藏 Turnstile input 的定向穿透 ---
  83. eles = self.driver.eles("tag:input")
  84. for ele in eles:
  85. attrs = getattr(ele, "attrs", {})
  86. name = attrs.get("name", "")
  87. inp_type = attrs.get("type", "")
  88. if "turnstile" in name and inp_type == "hidden":
  89. parent = ele.parent()
  90. if not parent:
  91. continue
  92. # 兼容不同版本 Turnstile 容器结构:
  93. # A. parent 本身挂载 shadow_root
  94. # B. parent 内部的前置同级 <div> 挂载 shadow_root
  95. host_candidates = [parent]
  96. sibling_div = parent.ele("tag:div", timeout=0.3)
  97. if sibling_div:
  98. host_candidates.append(sibling_div)
  99. for candidate in host_candidates:
  100. button = self._extract_turnstile_button_from_host(candidate)
  101. if button:
  102. self.log_message("Target verification button located via Turnstile host div.")
  103. return button
  104. # --- 策略 2: 备用递归深度搜索 (DFS) ---
  105. if dfs:
  106. self.log_message("Basic search missed. Initiating fallback DFS...")
  107. body = self.driver.ele("tag:body")
  108. if body:
  109. iframe = self.search_recursively_shadow_root_with_iframe(body)
  110. if iframe:
  111. iframe_body = iframe("tag:body")
  112. if iframe_body:
  113. return self.search_recursively_shadow_root_with_cf_input(iframe_body)
  114. return None
  115. except Exception as e:
  116. self.log_message(f"Error locating verification button: {e}")
  117. return None
  118. def search_recursively_shadow_root_with_iframe(
  119. self, ele: Optional[ChromiumElement], depth: int = 0, max_depth: int = 3
  120. ) -> Optional[ChromiumElement]:
  121. """递归检索挂载在 shadow_root 下的 iframe"""
  122. if depth > max_depth or not ele:
  123. return None
  124. try:
  125. sr = getattr(ele, "shadow_root", None)
  126. if sr:
  127. child = sr.child()
  128. if child and getattr(child, "tag", "") == "iframe":
  129. return child
  130. else:
  131. children = getattr(ele, "children", None)
  132. if callable(children):
  133. for child in children():
  134. result = self.search_recursively_shadow_root_with_iframe(child, depth + 1, max_depth)
  135. if result:
  136. return result
  137. except Exception:
  138. pass
  139. return None
  140. def search_recursively_shadow_root_with_cf_input(
  141. self, ele: Optional[ChromiumElement]
  142. ) -> Optional[ChromiumElement]:
  143. """递归检索挂载在 shadow_root 下的目标 input 按钮"""
  144. if not ele:
  145. return None
  146. try:
  147. sr = getattr(ele, "shadow_root", None)
  148. if sr:
  149. inp = sr.ele("tag:input")
  150. if inp:
  151. return inp
  152. else:
  153. children = getattr(ele, "children", None)
  154. if callable(children):
  155. for child in children():
  156. result = self.search_recursively_shadow_root_with_cf_input(child)
  157. if result:
  158. return result
  159. except Exception:
  160. pass
  161. return None
  162. def click_verification_button(self, is_dfs: bool = False) -> bool:
  163. """
  164. 定位并执行点击 Cloudflare 验证按钮。
  165. :return: 是否成功点击
  166. """
  167. try:
  168. button = self.locate_cf_button(dfs=is_dfs)
  169. if not button:
  170. self.log_message("Verification button not found.")
  171. return False
  172. states = getattr(button, "states", None)
  173. displayed = getattr(states, "is_displayed", False) if states else False
  174. enabled = getattr(states, "is_enabled", False) if states else False
  175. if displayed and enabled:
  176. self.log_message("Verification button clickable. Executing click...")
  177. button.click()
  178. time.sleep(1.0)
  179. self.log_message("Click executed successfully.")
  180. return True
  181. else:
  182. self.log_message(f"Button found (<{button.tag}>) but unclickable (displayed={displayed}, enabled={enabled}).")
  183. return False
  184. except Exception as e:
  185. self.log_message(f"Error clicking verification button: {e}")
  186. return False
  187. def is_bypassed(self) -> bool:
  188. """
  189. 根据当前页面标题判断是否已经脱离 Cloudflare 挑战页面。
  190. """
  191. try:
  192. title = (getattr(self.driver, "title", "") or "").lower()
  193. return "just a moment" not in title and "请稍候" not in title
  194. except Exception as e:
  195. self.log_message(f"Error checking page title: {e}")
  196. return False
  197. def bypass(self, max_retry: int = 5, use_dfs: bool = False) -> bool:
  198. """
  199. 执行穿透循环直至挑战通过或达到最大重试次数。
  200. """
  201. for i in range(max_retry):
  202. if self.is_bypassed():
  203. self.log_message("Cloudflare challenge bypassed successfully.")
  204. return True
  205. self.log_message(f"Verification page detected. Attempting bypass (try {i + 1}/{max_retry})...")
  206. self.click_verification_button(is_dfs=use_dfs)
  207. time.sleep(2.0)
  208. final_bypassed = self.is_bypassed()
  209. if final_bypassed:
  210. self.log_message("Cloudflare challenge bypassed on final check.")
  211. else:
  212. self.log_message(f"[WARN] Failed to clear challenge after {max_retry} attempts.")
  213. return final_bypassed
  214. def handle_waiting_room(self, max_wait_sec: int = 3600) -> bool:
  215. """
  216. 检测并等待 Cloudflare 等候室(Waiting Room)自动刷新放行。
  217. """
  218. wait_start = time.time()
  219. while time.time() - wait_start < max_wait_sec:
  220. try:
  221. html = (getattr(self.driver, "html", "") or "").lower()
  222. if "cloudflare" not in html:
  223. return True
  224. if any(w in html for w in ["file d'attente", "waitting room", "waiting room"]):
  225. self.log_message("In Waiting Room... Waiting for queue to refresh (sleeping 10s)...")
  226. time.sleep(10.0)
  227. else:
  228. return True
  229. except Exception as e:
  230. self.log_message(f"Error checking html in waiting room: {e}")
  231. break
  232. return True