Jelajahi Sumber

fix(logging): add 30s spawn failure backoff and clean up verbose/spammy logs across all modules

jerry 2 bulan lalu
induk
melakukan
03e6485f0f
7 mengubah file dengan 69 tambahan dan 50 penghapusan
  1. 12 2
      booker.py
  2. 6 5
      main.py
  3. 12 17
      plugins/ita_plugin.py
  4. 16 7
      plugins/vfs_plugin.py
  5. 6 1
      sentinel.py
  6. 2 2
      standalone.py
  7. 15 16
      utils/cloudflare_bypass_for_scraping.py

+ 12 - 2
booker.py

@@ -29,6 +29,7 @@ class BaseBookerGCO:
         self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
         self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
         self.m_last_group_query_time = 0.0
+        self.m_spawn_cooldown_until = 0.0
 
     def _log(self, message):
         prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
@@ -296,6 +297,9 @@ class BuiltinBookerGCO(BaseBookerGCO):
             try:
                 if not self._is_within_active_hours():
                     continue
+
+                if time.time() < self.m_spawn_cooldown_until:
+                    continue
                 
                 with self.m_lock:
                     current = len(self.m_tasks)
@@ -347,7 +351,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
             self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
         except Exception as e:
             err_str = str(e)
-            self._log(f"Spawn failed: {err_str}")
+            self.m_spawn_cooldown_until = time.time() + 30.0
+            self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
             rate_limited_indicators = [
                 "42901" in err_str,
                 "Rate limited" in err_str
@@ -523,6 +528,10 @@ class OrderBookerGCO(BaseBookerGCO):
             try:
                 if not self._is_within_active_hours():
                     continue
+
+                if time.time() < self.m_spawn_cooldown_until:
+                    continue
+
                 for apt in self.m_cfg.appointment_types:
                     r_key = apt.routing_key
                     with self.m_lock:
@@ -587,7 +596,8 @@ class OrderBookerGCO(BaseBookerGCO):
             self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
         except Exception as e:
             err_str = str(e)
-            self._log(f"Order Booker spawn failed: {err_str}")
+            self.m_spawn_cooldown_until = time.time() + 30.0
+            self._log(f"Order Booker spawn failed: {err_str}. Backing off for 30s...")
             rate_limited_indicators = [
                 "42901" in err_str,
                 "Rate limited" in err_str

+ 6 - 5
main.py

@@ -280,10 +280,11 @@ def main():
     sync_groups(role, groups_conf, wrappers, gco_classes, logger)
     logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
 
-    # 打印本地已安装插件列表与初始 RoleStatus
+    # 打印本地已安装插件列表与初始 RoleStatus 摘要
     initial_status = collect_role_status(role, node_id, wrappers)
-    status_dict = initial_status.to_dict()
-    logger.info("📊 Initial RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
+    installed_count = len(initial_status.installed_plugins)
+    active_count = len(wrappers)
+    logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
 
     report_ttl = max(180, int(args.status_interval * 3))
     try:
@@ -322,7 +323,7 @@ def main():
                             new_status = collect_role_status(role, node_id, wrappers)
                             status_dict = new_status.to_dict()
                             redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
-                            logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
+                            logger.info(f"📊 Reported updated RoleStatus: {len(new_status.groups)} group(s), {len(new_status.installed_plugins)} plugin(s).")
                 except queue.Empty:
                     break
 
@@ -333,7 +334,7 @@ def main():
                     current_status = collect_role_status(role, node_id, wrappers)
                     status_dict = current_status.to_dict()
                     redis_client.setex(status_key, report_ttl, json.dumps(status_dict))
-                    logger.info("📊 Reported RoleStatus:\n" + json.dumps(status_dict, indent=2, ensure_ascii=False))
+                    logger.info(f"📊 Reported RoleStatus: {len(current_status.groups)} group(s), {len(current_status.installed_plugins)} plugin(s).")
                 except Exception as e:
                     logger.warning(f"Error reporting RoleStatus to Redis '{status_key}': {e}")
 

+ 12 - 17
plugins/ita_plugin.py

@@ -302,19 +302,19 @@ class ItaPlugin(IVSPlg):
             elif self.page.ele('@name=callback_1', timeout=1):
                 self._log("[State] PingID Login Form detected. Submitting credentials...")
                 
-                print("正在输入账号...")
+                self._log("Entering username...")
                 user_input = self.page.ele('@name=callback_1')
                 self.mouse.human_click_ele(user_input)
                 user_input.input(username, clear=True)
                 time.sleep(1) # 模拟人手停顿
                 
-                print("正在输入密码...")
+                self._log("Entering password...")
                 pwd_input = self.page.ele('@type=password')
                 self.mouse.human_click_ele(pwd_input)
                 pwd_input.input(password, clear=True)
                 time.sleep(1)
                 
-                print("正在点击登录按钮...")
+                self._log("Clicking submit login button...")
                 submit_btn = self.page.ele('tag:button@type=submit') 
                 self.mouse.human_click_ele(submit_btn)
                 
@@ -368,16 +368,16 @@ class ItaPlugin(IVSPlg):
         # ==========================================
         # 步骤 1:在服务列表中寻找并点击目标服务的 Book
         # ==========================================
-        print("等待服务列表加载...")
+        self._log("Waiting for service list to load...")
         self.page.wait.eles_loaded('@aria-controls=dataTableServices', timeout=10)
 
-        print(f"正在查找 [{TARGET_SERVICE}] 的 Book 按钮...")
+        self._log(f"Searching for [{TARGET_SERVICE}] Book button...")
         # 使用 XPath:找包含 TARGET_SERVICE 文本的行(tr),再找该行里包含 Book 的超链接(a)
         target_book_btn_xpath = f'xpath://tr[contains(., "{TARGET_SERVICE}")]//a[contains(text(), "Book")]'
         book_btn = self.page.ele(target_book_btn_xpath)
 
         if not book_btn:
-            print(f"未找到 {TARGET_SERVICE} 的 Book 按钮,可能当前无号,程序退出。")
+            self._log(f"Target [{TARGET_SERVICE}] Book button not found, currently no slots available.")
             # 这里可以根据你的逻辑 return 或者 raise Exception
         else:
             self.mouse.human_click_ele(book_btn)
@@ -388,21 +388,19 @@ class ItaPlugin(IVSPlg):
         # ==========================================
         # 步骤 2:填写预约表单
         # ==========================================
-        print("等待预约表单加载...")
+        self._log("Waiting for booking form to load...")
         self.page.wait.eles_loaded('#bookingForm', timeout=10)
 
-        print("1. 选择预约类型...")
+        self._log("Selecting booking type & reason...")
         typeofbooking_ddl = self.page.ele('#typeofbookingddl')
         # DrissionPage select 方法直接按文本选中,会自动触发网页的 JS 联动
         typeofbooking_ddl.select('Individual booking')
         time.sleep(1)
 
-        print("2. 选择到访原因...")
         reason_ddl = self.page.ele('#ddls_0')
         reason_ddl.select(REASON_FOR_VISIT)
         time.sleep(1)
 
-        print("3. 填写备注...")
         notes_input = self.page.ele('#BookingNotes')
         notes_input.clear()
         notes_input.input('N/A') # 选填,填入 N/A 或者留空
@@ -411,16 +409,14 @@ class ItaPlugin(IVSPlg):
         # ==========================================
         # 步骤 3:处理 OTP 验证码
         # ==========================================
-        print("4. 点击发送 OTP 验证码...")
+        self._log("Sending OTP code...")
         otp_send_btn = self.page.ele('#otp-send')
         self.mouse.human_click_ele(otp_send_btn)
 
         # 这是一个 AJAX 请求,会转圈圈。我们需要等待成功提示出现
-        print("等待验证码发送成功的提示...")
         self.page.wait.ele_displayed('#IdOtpSent', timeout=15) # 等待绿字 "New code sent!" 显示
-        print("验证码已发送!")
+        self._log("OTP code sent successfully.")
 
-        print("5. 填写默认验证码 123456 ...")
         otp_input = self.page.ele('#otp-input')
         self.mouse.human_click_ele(otp_input)
         otp_input.input('123456')
@@ -429,20 +425,19 @@ class ItaPlugin(IVSPlg):
         # ==========================================
         # 步骤 4:勾选隐私政策并提交
         # ==========================================
-        print("6. 勾选隐私政策...")
         privacy_checkbox = self.page.ele('#PrivacyCheck')
         # 判断一下如果没勾上才去点,防止重复点击取消了
         if not privacy_checkbox.states.is_checked:
             self.mouse.human_click_ele(privacy_checkbox)
         time.sleep(1)
 
-        print("7. 点击 Forward 提交表单...")
+        self._log("Submitting forward form...")
         forward_btn = self.page.ele('#btnAvanti')
         self.mouse.human_click_ele(forward_btn)
 
         # 提交后页面会跳转,等待加载开始
         self.page.wait.load_start()
-        print("表单已提交!当前页面:", self.page.url)
+        self._log(f"Form submitted. Current page: {self.page.url}")
 
         time.sleep(5)
 

+ 16 - 7
plugins/vfs_plugin.py

@@ -287,6 +287,7 @@ class VfsPlugin(IVSPlg):
         cf_bypasser.handle_waiting_room()
         
         cf_token = ""
+        form_logged = False
         for i in range(40):
             time.sleep(1)
             self._handle_cookie_banner()
@@ -307,8 +308,10 @@ class VfsPlugin(IVSPlg):
                     pass
             
             if self.page.ele('tag:form', timeout=0.5) or self.page.ele('#mat-input-0', timeout=0.5):
-                self._log("Login form detected.")
-                if i > 5 and not cf_token:
+                if not form_logged:
+                    self._log("Login form detected.")
+                    form_logged = True
+                if i > 10 and not cf_token and not form_logged:
                     self._log("Form visible but token not found yet...")
 
         if not cf_token:
@@ -429,9 +432,12 @@ class VfsPlugin(IVSPlg):
                 fetch_options['body'] = data
                 log_body = str(data)
 
-        self._log(f"┌── [TRAFFIC REQUEST] {method} {req_url}")
-        self._log(f"├── Headers: {json.dumps(final_headers)}")
-        self._log(f"└── Body: {log_body}")
+        if self.config.debug:
+            self._log(f"┌── [TRAFFIC REQUEST] {method} {req_url}")
+            self._log(f"├── Headers: {json.dumps(final_headers)}")
+            self._log(f"└── Body: {log_body}")
+        else:
+            self._log(f"[TRAFFIC REQUEST] {method} {req_url}")
 
         js_script = f"""
         const url = "{req_url}";
@@ -476,8 +482,11 @@ class VfsPlugin(IVSPlg):
         duration = res_dict.get('duration', 0)
         resp_preview = resp.text[:1000] + "..." if len(resp.text) > 1000 else resp.text
         
-        self._log(f"┌── [TRAFFIC RESPONSE] Status: {resp.status_code} | Time: {duration}ms")
-        self._log(f"└── Body: {resp_preview}")
+        if self.config.debug:
+            self._log(f"┌── [TRAFFIC RESPONSE] Status: {resp.status_code} | Time: {duration}ms")
+            self._log(f"└── Body: {resp_preview}")
+        else:
+            self._log(f"[TRAFFIC RESPONSE] {method} {req_url} -> Status: {resp.status_code} ({duration}ms)")
 
         if resp.status_code == 200:
             return resp

+ 6 - 1
sentinel.py

@@ -21,6 +21,7 @@ class SentinelGCO:
         self.m_stop_event = threading.Event()
         self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
         self.m_last_group_query_time = 0.0
+        self.m_spawn_cooldown_until = 0.0
 
     def _log(self, message):
         if self.m_logger:
@@ -229,6 +230,9 @@ class SentinelGCO:
             try:
                 if not self._is_within_active_hours():
                     continue
+
+                if time.time() < self.m_spawn_cooldown_until:
+                    continue
                 
                 with self.m_lock:
                     current = len(self.m_tasks)
@@ -274,7 +278,8 @@ class SentinelGCO:
             self._log(f"+++ Sentinel spawned: {plg_cfg.account.username}")
         except Exception as e:
             err_str = str(e)
-            self._log(f"Spawn failed: {err_str}")
+            self.m_spawn_cooldown_until = time.time() + 30.0
+            self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
             rate_limited_indicators = [
                 "42901" in err_str,
                 "Rate limited" in err_str

+ 2 - 2
standalone.py

@@ -223,8 +223,8 @@ class StandaloneGCO:
                     try:
                         self._init_instance()
                     except Exception as e:
-                        self._log(f"Failed to create session: {e}. Retrying in 10s...")
-                        self.m_stop_event.wait(10.0)
+                        self._log(f"Failed to create session: {e}. Retrying in 30s...")
+                        self.m_stop_event.wait(30.0)
                         continue
 
                 # 2. 健康检查

+ 15 - 16
utils/cloudflare_bypass_for_scraping.py

@@ -3,17 +3,18 @@ from DrissionPage import ChromiumPage
 
 
 class CloudflareBypasser:
-    def __init__(self, driver: ChromiumPage, log=True):
+    def __init__(self, driver: ChromiumPage, log=False, logger=None):
         self.driver = driver
         self.log = log
+        self.logger = logger
         
     def log_message(self, message):
         if self.log:
-            print(message)
+            if self.logger:
+                self.logger(f"[CF-Bypass] {message}")
     
     def search_recursively_shadow_root_with_iframe(self, ele, depth=0, max_depth=3):
         if depth > max_depth:
-            self.log_message(f"Max depth {max_depth} reached during iframe search.")
             return None
 
         try:
@@ -26,11 +27,11 @@ class CloudflareBypasser:
                     result = self.search_recursively_shadow_root_with_iframe(child, depth + 1, max_depth)
                     if result:
                         return result
-        except Exception as e:
+        except Exception:
             pass 
         return None
 
-    def search_recursively_shadow_root_with_cf_input(self,ele):
+    def search_recursively_shadow_root_with_cf_input(self, ele):
         if ele.shadow_root:
             if ele.shadow_root.ele("tag:input"):
                 return ele.shadow_root.ele("tag:input")
@@ -57,17 +58,15 @@ class CloudflareBypasser:
                 return button
             else:
                 if dfs:
-                    self.log_message("Basic search failed. Searching for button recursively.")
                     ele = self.driver.ele("tag:body")
                     iframe = self.search_recursively_shadow_root_with_iframe(ele)
                     if iframe:
                         return self.search_recursively_shadow_root_with_cf_input(iframe("tag:body"))
-                    else:
-                        self.log_message("Iframe not found. Button search failed.")
             return None
 
         except Exception as e:
-            self.log_message(f"Error locating verification button: {e}")
+            if self.log:
+                self.log_message(f"Error locating verification button: {e}")
             return None
     
     def click_verification_button(self, is_dfs):
@@ -80,25 +79,24 @@ class CloudflareBypasser:
                     time.sleep(1)
                 else:
                     self.log_message("Button is not clickable.")
-            else:
-                self.log_message("Verification button not found.")
-
         except Exception as e:
-            self.log_message(f"Error clicking verification button: {e}")
+            if self.log:
+                self.log_message(f"Error clicking verification button: {e}")
 
     def is_bypassed(self):
         try:
             title = self.driver.title.lower()
             return "just a moment" not in title and "请稍候" not in title
         except Exception as e:
-            self.log_message(f"Error checking page title: {e}")
+            if self.log:
+                self.log_message(f"Error checking page title: {e}")
         return False
 
     def bypass(self, max_retry=5):
         for i in range(max_retry):
             if self.is_bypassed():
                 return True
-            self.log_message(f"Verification page detected.  Trying to bypass times={i}...")
+            self.log_message(f"Verification page detected. Trying to bypass times={i}...")
             self.click_verification_button(False)
             time.sleep(2)
         return self.is_bypassed()
@@ -119,5 +117,6 @@ class CloudflareBypasser:
                 else:
                     break
             except Exception as e:
-                self.log_message(f"Error checking html: {e}")
+                if self.log:
+                    self.log_message(f"Error checking html: {e}")