cloudflare_bypass_for_scraping.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. """
  2. Cloudflare 挑战自动化穿透工具模块。
  3. 针对 Cloudflare Turnstile / 5秒盾(Managed Challenge)提供高效、稳定的穿透能力。
  4. 核心设计理念:
  5. 1. 快速精准定位:定向穿透隐藏 turnstile input 对应的 shadow_root 宿主容器及嵌套 iframe,
  6. 无需遍历全局 DOM,执行耗时 < 0.2s。
  7. 2. 异常自愈回退:提供可控深度的 DFS 备用探测策略。
  8. 3. 等待室自适应排队:支持 Waiting Room 自动轮询处理。
  9. 4. 代码风格标准:严格遵循 Pythonic 规范,杜绝使用 hasattr,保证属性访问安全与可维护性。
  10. """
  11. import time
  12. from typing import Optional, Callable
  13. from DrissionPage import ChromiumPage
  14. from DrissionPage._elements.chromium_element import ChromiumElement
  15. class CloudflareBypasser:
  16. """
  17. Cloudflare 页面挑战识别与穿透处理器
  18. """
  19. def __init__(
  20. self,
  21. driver: ChromiumPage,
  22. log: bool = False,
  23. logger: Optional[Callable[[str], None]] = None
  24. ) -> None:
  25. self.driver = driver
  26. self.log = log
  27. self.logger = logger
  28. def log_message(self, message: str) -> None:
  29. """统一日志输出辅助方法"""
  30. if not self.log:
  31. return
  32. formatted = f"[CF-Bypass] {message}"
  33. if self.logger:
  34. if callable(self.logger):
  35. try:
  36. self.logger(formatted)
  37. except TypeError:
  38. self.logger(message)
  39. else:
  40. log_fn = getattr(self.logger, "info", print)
  41. log_fn(formatted)
  42. else:
  43. print(formatted)
  44. def _extract_turnstile_button_from_host(self, host_ele: Optional[ChromiumElement]) -> Optional[ChromiumElement]:
  45. """
  46. 从挂载 Shadow Root 的宿主容器元素中提取 Turnstile 复选框按钮。
  47. DOM 层级链路:
  48. Host Element (div)
  49. └─ shadow_root
  50. └─ child (iframe)
  51. └─ body
  52. └─ shadow_root
  53. └─ input[type=checkbox]
  54. """
  55. if not host_ele:
  56. return None
  57. orig_timeout = getattr(getattr(self.driver, "timeouts", None), "base", 2)
  58. try:
  59. if hasattr(self.driver, "set") and hasattr(self.driver.set, "timeouts"):
  60. self.driver.set.timeouts(base=0.3)
  61. host_sr = getattr(host_ele, "shadow_root", None)
  62. if not host_sr:
  63. return None
  64. iframe = host_sr.child(timeout=0.3)
  65. if not iframe:
  66. return None
  67. body = iframe("tag:body", timeout=0.3)
  68. if not body:
  69. return None
  70. body_sr = getattr(body, "shadow_root", None)
  71. if not body_sr:
  72. return None
  73. return body_sr("tag:input", timeout=0.3)
  74. except Exception as e:
  75. self.log_message(f"Error extracting button from host element: {e}")
  76. return None
  77. finally:
  78. if hasattr(self.driver, "set") and hasattr(self.driver.set, "timeouts"):
  79. self.driver.set.timeouts(base=orig_timeout)
  80. def locate_cf_button(self, dfs: bool = False) -> Optional[ChromiumElement]:
  81. """
  82. 定位 Cloudflare 验证复选框按钮。
  83. 策略 1(主路径,毫秒级):扫描主文档中的 'cf-turnstile-response' 隐藏 input,
  84. 其前置同级兄弟 div 或父级容器即为真实 shadow_root 宿主,直接提取目标按钮。
  85. 策略 2(备用路径):若主路径未命中且显式开启 dfs=True,则从 body 递归搜索。
  86. """
  87. try:
  88. # --- 策略 1: 基于隐藏 Turnstile input 的定向穿透 ---
  89. eles = self.driver.eles("tag:input")
  90. for ele in eles:
  91. attrs = getattr(ele, "attrs", {})
  92. name = attrs.get("name", "")
  93. inp_type = attrs.get("type", "")
  94. if "turnstile" in name and inp_type == "hidden":
  95. parent = ele.parent()
  96. if not parent:
  97. continue
  98. # 优先检查同级前置兄弟 <div>(在大多数现代 Turnstile 结构中该 div 才是 shadow_root 宿主)
  99. host_candidates = []
  100. sibling_div = parent.ele("tag:div", timeout=0.3)
  101. if sibling_div:
  102. host_candidates.append(sibling_div)
  103. host_candidates.append(parent)
  104. for candidate in host_candidates:
  105. button = self._extract_turnstile_button_from_host(candidate)
  106. if button:
  107. self.log_message("Target verification button located via Turnstile host div.")
  108. return button
  109. # --- 策略 2: 备用递归深度搜索 (DFS) ---
  110. if dfs:
  111. self.log_message("Basic search missed. Initiating fallback DFS...")
  112. body = self.driver.ele("tag:body")
  113. if body:
  114. iframe = self.search_recursively_shadow_root_with_iframe(body)
  115. if iframe:
  116. iframe_body = iframe("tag:body")
  117. if iframe_body:
  118. return self.search_recursively_shadow_root_with_cf_input(iframe_body)
  119. return None
  120. except Exception as e:
  121. self.log_message(f"Error locating verification button: {e}")
  122. return None
  123. def search_recursively_shadow_root_with_iframe(
  124. self, ele: Optional[ChromiumElement], depth: int = 0, max_depth: int = 3
  125. ) -> Optional[ChromiumElement]:
  126. """递归检索挂载在 shadow_root 下的 iframe"""
  127. if depth > max_depth or not ele:
  128. return None
  129. try:
  130. sr = getattr(ele, "shadow_root", None)
  131. if sr:
  132. child = sr.child()
  133. if child and getattr(child, "tag", "") == "iframe":
  134. return child
  135. else:
  136. children = getattr(ele, "children", None)
  137. if callable(children):
  138. for child in children():
  139. result = self.search_recursively_shadow_root_with_iframe(child, depth + 1, max_depth)
  140. if result:
  141. return result
  142. except Exception:
  143. pass
  144. return None
  145. def search_recursively_shadow_root_with_cf_input(
  146. self, ele: Optional[ChromiumElement]
  147. ) -> Optional[ChromiumElement]:
  148. """递归检索挂载在 shadow_root 下的目标 input 按钮"""
  149. if not ele:
  150. return None
  151. try:
  152. sr = getattr(ele, "shadow_root", None)
  153. if sr:
  154. inp = sr.ele("tag:input")
  155. if inp:
  156. return inp
  157. else:
  158. children = getattr(ele, "children", None)
  159. if callable(children):
  160. for child in children():
  161. result = self.search_recursively_shadow_root_with_cf_input(child)
  162. if result:
  163. return result
  164. except Exception:
  165. pass
  166. return None
  167. def click_verification_button(self, is_dfs: bool = False) -> bool:
  168. """
  169. 定位并执行点击 Cloudflare 验证按钮。
  170. :return: 是否成功点击
  171. """
  172. try:
  173. button = self.locate_cf_button(dfs=is_dfs)
  174. if not button:
  175. self.log_message("Verification button not found.")
  176. return False
  177. states = getattr(button, "states", None)
  178. displayed = getattr(states, "is_displayed", False) if states else False
  179. enabled = getattr(states, "is_enabled", False) if states else False
  180. if displayed and enabled:
  181. self.log_message("Verification button clickable. Executing click...")
  182. button.click()
  183. time.sleep(1.0)
  184. self.log_message("Click executed successfully.")
  185. return True
  186. else:
  187. self.log_message(f"Button found (<{button.tag}>) but unclickable (displayed={displayed}, enabled={enabled}).")
  188. return False
  189. except Exception as e:
  190. self.log_message(f"Error clicking verification button: {e}")
  191. return False
  192. def is_bypassed(self) -> bool:
  193. """
  194. 根据当前页面标题判断是否已经脱离 Cloudflare 挑战页面。
  195. """
  196. try:
  197. title = (getattr(self.driver, "title", "") or "").lower()
  198. return "just a moment" not in title and "请稍候" not in title
  199. except Exception as e:
  200. self.log_message(f"Error checking page title: {e}")
  201. return False
  202. def bypass(self, max_retry: int = 5, use_dfs: bool = False) -> bool:
  203. """
  204. 执行穿透循环直至挑战通过或达到最大重试次数。
  205. """
  206. for i in range(max_retry):
  207. if self.is_bypassed():
  208. self.log_message("Cloudflare challenge bypassed successfully.")
  209. return True
  210. self.log_message(f"Verification page detected. Attempting bypass (try {i + 1}/{max_retry})...")
  211. self.click_verification_button(is_dfs=use_dfs)
  212. time.sleep(2.0)
  213. final_bypassed = self.is_bypassed()
  214. if final_bypassed:
  215. self.log_message("Cloudflare challenge bypassed on final check.")
  216. else:
  217. self.log_message(f"[WARN] Failed to clear challenge after {max_retry} attempts.")
  218. return final_bypassed
  219. def handle_waiting_room(self, max_wait_sec: int = 3600) -> bool:
  220. """
  221. 检测并等待 Cloudflare 等候室(Waiting Room)自动刷新放行。
  222. """
  223. wait_start = time.time()
  224. while time.time() - wait_start < max_wait_sec:
  225. try:
  226. html = (getattr(self.driver, "html", "") or "").lower()
  227. if "cloudflare" not in html:
  228. return True
  229. if any(w in html for w in ["file d'attente", "waitting room", "waiting room"]):
  230. self.log_message("In Waiting Room... Waiting for queue to refresh (sleeping 10s)...")
  231. time.sleep(10.0)
  232. else:
  233. return True
  234. except Exception as e:
  235. self.log_message(f"Error checking html in waiting room: {e}")
  236. break
  237. return True