Explorar el Código

test: add resource lifecycle tests and utilities for robust cleanup and process termination

jerry hace 3 semanas
padre
commit
82fab8ab14

+ 2 - 0
.dockerignore

@@ -23,6 +23,8 @@ env/
 .env.*
 *.local
 secrets.json
+**/secrets.json
+data/booking_outbox.sqlite3*
 
 # —— 运行时日志、会话与临时数据 ——
 logs/

+ 3 - 1
.gitignore

@@ -31,6 +31,7 @@ env/
 !.env.example
 *.local
 secrets.json
+data/booking_outbox.sqlite3*
 config/ita_query_account.local.json
 config/test_conf.local.json
 
@@ -97,4 +98,5 @@ powershell.cmd
 !config/config_booker.yaml
 !config/config_standalone.yaml
 data/supersaas_accounts.*
-
+*.onnx
+data/*.onnx

+ 55 - 72
README.md

@@ -2,6 +2,8 @@
 
 Coordinate 是一个高并发、插件化的多国签证预约与监控自动化系统。支持通过声明式状态机与拟人化浏览器自动化(DrissionPage / curl_cffi / 大模型视觉识别 / CapSolver 等)完成反爬穿透、会话建立、号源实时监控与自动占位预约。
 
+调度并发、订单恢复、成功结果持久化与服务凭据配置见 [调度可靠性与运行配置](docs/scheduler-reliability.md)。
+
 ---
 
 ## 目录
@@ -9,7 +11,7 @@ Coordinate 是一个高并发、插件化的多国签证预约与监控自动化
   - [7 步标准化测试流水线](#7-步标准化测试流水线)
   - [CLI 命令行常用方法与示例](#cli-命令行常用方法与示例)
   - [命令行全量参数速查表](#命令行全量参数速查表)
-  - [测试配置文件说明 (`config/test_conf.json`)](#测试配置文件说明-configtest_confjson)
+  - [测试配置文件说明 (`config/test_conf.yaml`)](#测试配置文件说明-configtest_confyaml)
   - [诊断报告解读与排错](#诊断报告解读与排错)
   - [France-Visas (法签申请表) 账号全自动注册、填表与 FRA1 提取](#france-visas-法签申请表-账号全自动注册填表与-fra1-提取)
   - [核心自动化流水线](#1-核心自动化流水线)
@@ -105,9 +107,9 @@ python3 tools/plugin_tester.py --plugin pol_plugin \
 ```
 
 #### 7. 指定特定配置文件
-默认自动加载 `config/test_conf.json`,可通过 `-c` 指定自定义配置文件:
+默认自动加载 `config/test_conf.yaml`,可通过 `-c` 指定自定义配置文件:
 ```bash
-python3 tools/plugin_tester.py --plugin vfs_plugin -c "config/staging_conf.json"
+python3 tools/plugin_tester.py --plugin vfs_plugin -c "config/staging_conf.yaml"
 ```
 
 #### 8. 全量批量回归测试
@@ -126,7 +128,7 @@ python3 tools/plugin_tester.py --plugin all
 | `--no-cleanup` | flag | `False` | 测试结束后不自动执行 `cleanup()`,保持浏览器打开 |
 | `--book` | flag | `False` | 是否执行预约提交流程(`test_book`),默认仅执行到查询步骤 |
 | `--only-init` | flag | `False` | 仅验证插件类装载与配置注入后立即退出 |
-| `--config` / `-c` | string | `config/test_conf.json` | 指定自定义 JSON 测试配置文件路径 |
+| `--config` / `-c` | string | `config/test_conf.yaml` | 指定自定义 YAML 测试配置文件路径 |
 | `--group` / `-g` | string | `None` | 覆盖插件分组标识 `group_id` |
 | `--user` / `-u` | string | `None` | 覆盖测试账号用户名 / 邮箱 |
 | `--password` / `-pwd` | string | `None` | 覆盖测试账号密码 |
@@ -134,73 +136,54 @@ python3 tools/plugin_tester.py --plugin all
 
 ---
 
-### 测试配置文件说明 (`config/test_conf.json`)
+### 测试配置文件说明 (`config/test_conf.yaml`)
 
 配置文件集中管理各签证中心的站点参数、测试账号及申请人信息:
 
-```json
-{
-  "default": {
-    "account": {
-      "username": "tester@example.com",
-      "password": "SamplePassword123!"
-    },
-    "proxy": {
-      "proto": "http",
-      "ip": "127.0.0.1",
-      "port": 7890,
-      "username": "",
-      "password": ""
-    },
-    "session_max_life": 3600,
-    "debug": true
-  },
-  "plugins": {
-    "vfs_plugin": {
-      "group_id": "vfs.cn.at",
-      "account": {
-        "username": "cn_at_wfvl5y@text.skin",
-        "password": "SamplePassword$"
-      },
-      "appointment_types": [
-        {
-          "city": "Beijing",
-          "country": "Austria",
-          "visa_type": "Tourist",
-          "routing_key": "slot.bjs.at.tourist",
-          "weight": 10
-        }
-      ],
-      "free_config": {
-        "language": "en",
-        "country_code": "chn",
-        "mission_code": "aut",
-        "apt_configs": {
-          "slot.bjs.at.tourist": {
-            "vac_code": "Bei",
-            "category_code": "VTC",
-            "subcategory_code": "TR"
-          }
-        }
-      },
-      "user_inputs": {
-        "first_name": "San",
-        "last_name": "Zhang",
-        "email": "applicant@example.com",
-        "expected_start_date": "2026-09-01",
-        "expected_end_date": "2026-11-30"
-      }
-    },
-    "pol_plugin": {
-      "group_id": "pol.dublin",
-      "free_config": {
-        "query_url": "https://secure.e-konsulat.gov.pl/placowki/151/wiza-schengen/wizyty/weryfikacja-obrazkowa",
-        "service_type": "Wiza Schengen",
-        "location": "Dublin"
-      }
-    }
-  }
-}
+```yaml
+default:
+  account:
+    username: tester@example.com
+    password: SamplePassword123!
+  proxy_pool:
+    - local:local_multi_pool
+  session_max_life: 3600
+  debug: true
+
+plugins:
+  vfs_plugin:
+    group_id: vfs.cn.at
+    account:
+      username: cn_at_wfvl5y@text.skin
+      password: SamplePassword$
+    appointment_types:
+      - city: Beijing
+        country: Austria
+        visa_type: Tourist
+        routing_key: slot.bjs.at.tourist
+        weight: 10
+    free_config:
+      language: en
+      country_code: chn
+      mission_code: aut
+      apt_configs:
+        slot.bjs.at.tourist:
+          vac_code: Bei
+          category_code: VTC
+          subcategory_code: TR
+    user_inputs:
+      first_name: San
+      last_name: Zhang
+      email: applicant@example.com
+      expected_start_date: "2026-09-01"
+      expected_end_date: "2026-11-30"
+
+  pol_plugin:
+    group_id: pol.dublin
+    free_config:
+      query_url: https://secure.e-konsulat.gov.pl/placowki/151/wiza-schengen/wizyty/weryfikacja-obrazkowa
+      service_type: Wiza Schengen
+      location: Dublin
 ```
 
 ---
@@ -243,7 +226,7 @@ from tools.plugin_tester import PluginTester
 # 从配置文件自动加载并运行
 tester = PluginTester.from_json_config(
     plugin_name="pol_plugin",
-    config_path="config/test_conf.json"
+    config_path="config/test_conf.yaml"
 )
 all_passed = tester.run_pipeline(test_book=False, keep_on_error=True)
 
@@ -492,9 +475,9 @@ otp_code = breaker.break_captcha(image_data, prompt=custom_prompt)
 ```text
 coordinate/
 ├── config/                  # 运行时与测试配置目录
-│   ├── test_conf.json       # PluginTester 全插件测试参数集
-│   ├── config_booker.json   # 抢号预约进程配置
-│   └── config_sentinel.json # 巡检监控进程配置
+│   ├── test_conf.yaml       # PluginTester 全插件测试参数集
+│   ├── config_booker.yaml   # 抢号预约进程配置
+│   └── config_sentinel.yaml # 巡检监控进程配置
 ├── configure.py             # 核心系统配置(Redis、大模型 API、Mihomo 等)
 ├── plugins/                 # 签证中心业务插件 (实现 IPlugin 规范)
 │   ├── vfs_plugin.py        # VFS 插件

+ 236 - 129
booker.py

@@ -1,4 +1,5 @@
 import os
+from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
 import time
 import json
 import threading
@@ -16,7 +17,8 @@ from core_types import (
 from plugin_factory import PluginFactory 
 from toolkit.cloud_api import CloudApi
 from toolkit.proxy_manager import ProxyManager
-from toolkit.thread_pool import ThreadPool
+from toolkit.task_executor import TaskExecutor
+from utils.booking_outbox import BookingOutbox
 from toolkit.notifier import notify_book_result
 from utils.safe_redis_cli import SafeRedisClient
 
@@ -35,9 +37,16 @@ class BaseBookerGCO:
         self.m_lock = threading.RLock()
         self.m_stop_event = threading.Event()
         self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
+        init_lifecycle(self)
         self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
         self.m_last_group_query_time = 0.0
         self.m_spawn_cooldown_until = 0.0
+        self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking")
+        self._leased_orders = set()
+        self._pending_results = {}
+        self._outbox = BookingOutbox()
+        self._background_threads = []
+        self._heartbeat_stop_event = threading.Event()
 
     def _log(self, message):
         prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
@@ -72,6 +81,7 @@ class BaseBookerGCO:
             updated_at=utc_now_str(),
         )
 
+    @start_once
     def start(self):
         if not self.m_cfg.enable:
             return
@@ -81,19 +91,42 @@ class BaseBookerGCO:
         plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
         self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
 
-        threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
-        threading.Thread(target=self._creator_loop, daemon=True).start()
-        threading.Thread(target=self._maintain_loop, daemon=True).start()
+        for target in (self._booking_trigger_loop, self._creator_loop,
+                       self._maintain_loop, self._heartbeat_loop, self._success_sync_loop):
+            thread = threading.Thread(target=target, daemon=True, name=target.__name__)
+            self._background_threads.append(thread)
+            thread.start()
         self._start_additional_threads()
 
     def _start_additional_threads(self):
         """Hook for subclasses to start additional background threads."""
         pass
 
+    @serialized_stop
     def stop(self):
         self._log("Stopping Booker...")
         self.m_stop_event.set()
-        self._cleanup_all_tasks("booker stop")
+        deadline = time.monotonic() + self.m_cfg.shutdown_timeout
+        self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
+        workers = [t for t in self._background_threads if t.name != '_heartbeat_loop']
+        join_threads(workers, max(0, deadline - time.monotonic()))
+        with self.m_lock:
+            for task in self.m_tasks:
+                task.removed = True
+                self._resources.track(task.instance)
+        self._resources.close(self.m_cfg.cleanup_timeout)
+        threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1)
+        if threads_done and not self._pending_results:
+            self._heartbeat_stop_event.set()
+            threads_done = join_threads(
+                [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2)
+        if not self._resources.empty or not threads_done or self._pending_results:
+            raise RuntimeError('Booker shutdown incomplete; resources retained for retry')
+        with self.m_lock:
+            self.m_tasks.clear()
+            self._on_all_tasks_cleaned()
+            self._leased_orders.clear()
+        self.redis_client.close()
 
     def update_config(self, new_cfg: GroupConfig):
         """Dynamically updates the configuration."""
@@ -108,24 +141,27 @@ class BaseBookerGCO:
             self.m_cfg = new_cfg
 
     def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
-        try:
-            if instance:
-                ref_str = f" for task={task_ref}" if task_ref else ""
-                self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
-                instance.cleanup()
-        except Exception as e:
-            self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
+        if instance is None:
+            return True
+        cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
+        if not cleaned:
+            self._log(f"Cleanup retained for retry ({reason})")
+        return cleaned
 
     def _cleanup_task(self, task: Task, reason: str = ""):
         if task and task.instance:
-            self._cleanup_instance(task.instance, reason, task.task_ref)
+            with task._operation_lock:
+                self._cleanup_instance(task.instance, reason, task.task_ref)
 
     def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
         removed = False
         with self.m_lock:
             if task in self.m_tasks:
+                task.removed = True
                 self.m_tasks.remove(task)
                 removed = True
+                if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
+                    self._leased_orders.discard(str(task.task_ref))
             self._on_task_removed(task)
                 
         if cleanup and removed:
@@ -140,6 +176,10 @@ class BaseBookerGCO:
         with self.m_lock:
             tasks = list(self.m_tasks)
             self.m_tasks.clear()
+            for task in tasks:
+                task.removed = True
+                if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
+                    self._leased_orders.discard(str(task.task_ref))
             self._on_all_tasks_cleaned()
         for task in tasks:
             self._cleanup_task(task, reason)
@@ -226,11 +266,8 @@ class BaseBookerGCO:
                     self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
                 except Exception as update_err:
                     self._log(f"Failed to update VAS task status: {update_err}")
-
-                try:
+                else:
                     self.redis_client.zrem(self.m_tracker_key, effective_task_id)
-                except Exception as zrem_err:
-                    self._log(f"Failed to remove task from tracker: {zrem_err}")
 
             # If pool account, permanently disable it in cloud
             if acc and acc.id > 0 and acc.username not in ("", "Guest"):
@@ -338,93 +375,143 @@ class BaseBookerGCO:
         else:
             return current_utc_time >= start_time or current_utc_time <= end_time
 
-    def _maintain_loop(self):
-        self._log("Maintain loop started.")
-        while not self.m_stop_event.is_set():
-            try:
-                time.sleep(1.0)
-                with self.m_lock:
-                    tasks_to_check = list(self.m_tasks)
-                
-                if not tasks_to_check:
-                    continue
-                
-                dead_tasks = []
-                healthy_tasks = []
-                now = time.time()
-                for t in tasks_to_check:
-                    if now >= t.next_remote_ping:
-                        t.instance.keep_alive()
-                        if t.instance.health_check():
-                            healthy_tasks.append(t)
-                            t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
-                        else:
-                            dead_tasks.append(t)
-                    else:
-                        healthy_tasks.append(t)
-                
-                self._on_maintain_ping(healthy_tasks, dead_tasks)
+    def _lease_order(self, task_id):
+        with self.m_lock:
+            self._leased_orders.add(str(task_id))
+            self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120})
 
-                if dead_tasks:
-                    for t in dead_tasks:
-                        self._remove_task(t, "unhealthy or keep-alive failed")
-            except Exception as e:
-                self._log(f'Maintain loop exception: {e}')
+    def _renew_leases(self):
+        with self.m_lock:
+            if self._leased_orders:
+                self.redis_client.bulk_zadd(self.m_tracker_key, {
+                    tid: time.time() + 120 for tid in self._leased_orders})
 
-    def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
-        """Subclass hook for additional handling of healthy/dead tasks during maintenance."""
-        pass
+    def _heartbeat_loop(self):
+        # No browser calls here: long OTP/HTTP waits must not stop order leases.
+        while not self._heartbeat_stop_event.wait(10):
+            self._renew_leases()
+
+    def _sync_successes(self):
+        try:
+            with self.m_lock:
+                pending = dict(self._pending_results)
+            for task_id, payload in pending.items():
+                saved = False
+                try:
+                    self._outbox.put(task_id, self.m_tracker_key, payload)
+                    saved = True
+                except Exception as exc:
+                    self._log(f"Local success journal unavailable for task={task_id}: {exc}")
+                saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved
+                if saved:
+                    with self.m_lock:
+                        self._pending_results.pop(task_id, None)
+                        self._leased_orders.discard(task_id)
+            self._outbox.flush(CloudApi.Instance(), self.redis_client._client,
+                               self._log, tracker=self.m_tracker_key)
+        except Exception as exc:
+            self._log(f"Success outbox retry failed: {exc}")
+
+    def _success_sync_loop(self):
+        while not self.m_stop_event.is_set() or self._pending_results:
+            self._sync_successes()
+            if self.m_stop_event.is_set() and not self._pending_results:
+                break
+            if self.m_stop_event.is_set():
+                self._heartbeat_stop_event.wait(5)
+            else:
+                self.m_stop_event.wait(5)
+
+    def _maintain_task(self, task):
+        try:
+            with task._operation_lock, self._resources.operation(task.instance):
+                if task.removed or self.m_stop_event.is_set():
+                    return
+                task.instance.keep_alive()
+                if task.instance.health_check():
+                    task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5))
+                else:
+                    self._remove_task(task, "unhealthy or keep-alive failed")
+        except Exception as exc:
+            task.next_remote_ping = time.time() + 5
+            self._dispatch_error_policy("keep_alive", exc, task=task)
+        finally:
+            with self.m_lock:
+                task.busy = False
+
+    def _schedule_maintenance(self):
+        with self.m_lock:
+            if self.m_stop_event.is_set():
+                return
+            for task in self.m_tasks:
+                if task.busy or task.removed or time.time() < task.next_remote_ping:
+                    continue
+                task.busy = True
+                if not self._executor.submit(self._maintain_task, task):
+                    task.busy = False
+
+    def _maintain_loop(self):
+        while not self.m_stop_event.wait(1):
+            self._schedule_maintenance()
 
     def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
         """Checks if available dates in query_result match task preferences (defaults to True)."""
         return True
 
+    def _run_book_job(self, task, query_result):
+        try:
+            with task._operation_lock, self._resources.operation(task.instance):
+                if not task.removed and not self.m_stop_event.is_set():
+                    self._execute_book_job(task, query_result)
+        finally:
+            with self.m_lock:
+                task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown)
+                task.busy = False
+
+    def _poll_booking_signals(self):
+        now = time.time()
+        with self.m_lock:
+            ready = [t for t in self.m_tasks if not t.busy and not t.removed
+                     and t.book_allowed and now >= t.next_run]
+            keys = {key for t in ready for key in t.acceptable_routing_keys}
+        apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys]
+        if not apt_types:
+            return
+        redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types]
+        signals = self.redis_client.mget(redis_keys)
+        for apt_type, raw_data in zip(apt_types, signals):
+            if not raw_data:
+                continue
+            try:
+                data = json.loads(raw_data)
+                result = QueryResult.model_validate(data['query_result'])
+                result.apt_type = AppointmentType.model_validate(data['apt_type'])
+                if result.apt_type.routing_key != apt_type.routing_key:
+                    raise ValueError('signal routing key mismatch')
+            except Exception as exc:
+                # A producer may have replaced this value since MGET. Do not
+                # delete a newer signal based on an old malformed payload.
+                self._log(f"Invalid signal for {apt_type.routing_key}: {exc}")
+                continue
+            with self.m_lock:
+                for task in ready:
+                    if (task.busy or task.removed or self.m_stop_event.is_set()
+                            or apt_type.routing_key not in task.acceptable_routing_keys
+                            or not self._is_date_of_interest(task, result)):
+                        continue
+                    task.busy = True
+                    if not self._executor.submit(self._run_book_job, task, result):
+                        task.busy = False
+
     def _booking_trigger_loop(self):
         self._log("Trigger loop started.")
         while not self.m_stop_event.is_set():
             try:
-                time.sleep(0.1)
-                now = time.time()
-                for apt_type in self.m_cfg.appointment_types:
-                    redis_key = self._get_redis_key(apt_type.routing_key)
-                    raw_data = self.redis_client.get(redis_key)
-                    if not raw_data:
-                        continue
-                    try:
-                        data = json.loads(raw_data)
-                        query_result = QueryResult.model_validate(data['query_result'])
-                        query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
-                    except Exception as parse_err:
-                        self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
-                        self.redis_client.delete(redis_key)
-                        continue
-                    
-                    matching_tasks = []
-                    with self.m_lock:
-                        for task in self.m_tasks:
-                            if now < task.next_run or not task.book_allowed:
-                                continue
-                            if apt_type.routing_key not in task.acceptable_routing_keys:
-                                continue
-                            if not self._is_date_of_interest(task, query_result):
-                                continue
-                            
-                            task.next_run = now + self.m_cfg.booker.booking_cooldown
-                            matching_tasks.append(task)
-                            
-                    if matching_tasks:
-                        threads = []
-                        for task in matching_tasks:
-                            self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
-                            t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
-                            threads.append(t)
-                            t.start()
-                        
-                        for t in threads:
-                            t.join() 
-                    
-            except Exception as e:
-                self._log(f"Booking trigger loop exception: {e}")
+                if self.m_cfg.enable and self._is_within_active_hours():
+                    self._poll_booking_signals()
+            except Exception as exc:
+                self._log(f"Booking trigger loop exception: {exc}")
+            self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval)
 
     def _execute_book_job(self, task: Task, query_result: QueryResult):
         raise NotImplementedError
@@ -444,15 +531,16 @@ class BaseBookerGCO:
             "payment_link": book_res.payment_link
         }
         
-        def _update_cloud_success():
-            try:
-                CloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
-            except Exception as e:
-                self._log(f"Failed to update success state to cloud: {e}")
+        payload = {"status": "grabbed", "grabbed_history": grab_info}
+        # Commit locally before retiring the browser or dropping its lease.
+        with self.m_lock:
+            self._pending_results[str(task_id)] = payload
+        self._sync_successes()
+        try:
+            notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
+        except Exception as exc:
+            self._log(f"Booking succeeded, but notification could not be queued: {exc}")
 
-        ThreadPool.getInstance().enqueue(_update_cloud_success)
-        notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
-        self.redis_client.zrem(self.m_tracker_key, task_id)
 
 
 class BuiltinBookerGCO(BaseBookerGCO):
@@ -479,6 +567,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
                 self._spawn_worker()
 
     def _spawn_worker(self):
+        if self.m_stop_event.is_set() or not self._resources.retry_failed():
+            return
         instance = None
         task = None
         success = False
@@ -503,9 +593,11 @@ class BuiltinBookerGCO(BaseBookerGCO):
                 plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
 
             instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
-            instance.set_log(self.m_logger)
-            instance.set_config(plg_cfg)
-            instance.create_session()
+            self._resources.track(instance)
+            with self._resources.operation(instance):
+                instance.set_log(self.m_logger)
+                instance.set_config(plg_cfg)
+                instance.create_session()
 
             all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
             task = Task(
@@ -518,6 +610,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
                 next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5) 
             )
             with self.m_lock:
+                if self.m_stop_event.is_set():
+                    return
                 self.m_tasks.append(task)
 
             success = True
@@ -554,7 +648,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
 
         task_id = task_data.get('id')
         order_id = task_data.get('order_id')
-        self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
+        self._lease_order(task_id)
+        book_res = None
                     
         try:
             user_input = task_data.get('user_inputs', {})
@@ -572,7 +667,13 @@ class BuiltinBookerGCO(BaseBookerGCO):
             else:
                 self._log(f"❌ BOOK FAILED for Order: {order_id}")
         except Exception as e:
-            self._dispatch_error_policy("book", e, task=task)
+            self._dispatch_error_policy("book", e, task=task, task_id=task_id)
+        finally:
+            with self.m_lock:
+                if str(task_id) not in self._pending_results:
+                    self._leased_orders.discard(str(task_id))
+            if book_res is None or not book_res.success:
+                self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()})
 
 
 class OrderBookerGCO(BaseBookerGCO):
@@ -587,36 +688,28 @@ class OrderBookerGCO(BaseBookerGCO):
     def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
         super().__init__(cfg, redis_conf, logger)
         self.m_task_data_cache: Dict[str, dict] = {}
-        self.heartbeat_ttl = 2 * 60.0
 
     def _start_additional_threads(self):
-        threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
+        thread = threading.Thread(target=self._cache_refresh_loop, daemon=True)
+        self._background_threads.append(thread)
+        thread.start()
 
     def _on_task_removed(self, task: Task):
         task_id = task.task_ref
-        if task_id:
+        if task_id is not None:
             self.m_task_data_cache.pop(str(task_id), None)
 
     def _on_all_tasks_cleaned(self):
         self.m_task_data_cache.clear()
 
-    def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
-        if healthy_tasks:
-            new_deadline = time.time() + self.heartbeat_ttl
-            mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
-            self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
-
-        if dead_tasks:
-            mapping = {str(t.task_ref): 0 for t in dead_tasks}
-            self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
-
     def _cache_refresh_loop(self):
         self._log("Cache refresh loop started.")
         refresh_interval = 15 * 60
         
         while not self.m_stop_event.is_set():
             try:
-                time.sleep(1)
+                if self.m_stop_event.wait(1):
+                    break
                 with self.m_lock:
                     tasks_to_check = {
                         tid: data.get('_last_refresh', 0) 
@@ -628,6 +721,8 @@ class OrderBookerGCO(BaseBookerGCO):
                 
                 now = time.time()
                 for tid, last_refresh in tasks_to_check.items():
+                    if self.m_stop_event.is_set():
+                        break
                     if now - last_refresh >= refresh_interval:
                         fresh_data = CloudApi.Instance().get_vas_task(tid)
                         if fresh_data:
@@ -636,7 +731,7 @@ class OrderBookerGCO(BaseBookerGCO):
                             with self.m_lock:
                                 if tid in self.m_task_data_cache:
                                     self.m_task_data_cache[tid] = fresh_data
-                        time.sleep(0.5)
+                        self.m_stop_event.wait(0.5)
             except Exception as e:
                 self._log(f'Cache refresh loop exception: {e}')
 
@@ -651,6 +746,8 @@ class OrderBookerGCO(BaseBookerGCO):
             or '2100-01-01'
         )
         available_date = query_result.earliest_date
+        if available_date is None:
+            return False
         dt = available_date.strftime("%Y-%m-%d")
         return dt <= expected_end_date
 
@@ -661,7 +758,7 @@ class OrderBookerGCO(BaseBookerGCO):
         try:
             with self.m_lock:
                 task_data = self.m_task_data_cache.get(str(task_id))
-            if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
+            if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']:
                 self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
                 self._remove_task(task, "bound task no longer valid")
                 self.redis_client.zrem(self.m_tracker_key, task_id)
@@ -699,6 +796,8 @@ class OrderBookerGCO(BaseBookerGCO):
                     self._spawn_worker(r_key)
 
     def _spawn_worker(self, target_routing_key: str):
+        if self.m_stop_event.is_set() or not self._resources.retry_failed():
+            return
         instance = None
         task = None
         success = False
@@ -720,7 +819,7 @@ class OrderBookerGCO(BaseBookerGCO):
             with self.m_lock:
                 self.m_task_data_cache[str(task_id)] = task_data
             
-            self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
+            self._lease_order(task_id)
             user_inputs = task_data.get('user_inputs', {})
             
             plg_cfg = PluginConfig()
@@ -742,9 +841,11 @@ class OrderBookerGCO(BaseBookerGCO):
                 plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
 
             instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
-            instance.set_log(self.m_logger)
-            instance.set_config(plg_cfg)
-            instance.create_session()
+            self._resources.track(instance)
+            with self._resources.operation(instance):
+                instance.set_log(self.m_logger)
+                instance.set_config(plg_cfg)
+                instance.create_session()
 
             task = Task(
                 instance=instance,
@@ -756,6 +857,8 @@ class OrderBookerGCO(BaseBookerGCO):
                 next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
             )
             with self.m_lock:
+                if self.m_stop_event.is_set():
+                    return
                 self.m_tasks.append(task)
 
             success = True             
@@ -773,10 +876,14 @@ class OrderBookerGCO(BaseBookerGCO):
         finally:
             if not success and instance:
                 self._cleanup_instance(instance, "spawn failed")
-            if not success and task_id:
+            if not success and task_id is not None:
                 with self.m_lock:
                     self.m_task_data_cache.pop(str(task_id), None)
                 try:
-                    self.redis_client.zrem(self.m_tracker_key, task_id)
+                    with self.m_lock:
+                        self._leased_orders.discard(str(task_id))
+                    # Retain a recovery deadline (including any 429 cooldown).
+                    self.redis_client.zadd(self.m_tracker_key, {
+                        str(task_id): max(time.time(), self.m_spawn_cooldown_until)})
                 except Exception:
-                    pass
+                    pass

+ 5 - 0
config/secrets.example.json

@@ -0,0 +1,5 @@
+{
+  "CLOUD_API_TOKEN": "",
+  "CAPTCHA_LLM_API_KEY": "",
+  "REDIS_PASSWORD": ""
+}

+ 28 - 5
configure.py

@@ -1,17 +1,40 @@
+import json
+import os
+from pathlib import Path
+
+
+def _local_secrets():
+    path = Path(__file__).resolve().parent / "config" / "secrets.json"
+    if not path.exists():
+        return {}
+    with path.open(encoding="utf-8") as stream:
+        return json.load(stream)
+
+
+_SECRETS = _local_secrets()
+
+
+def _setting(name, default=""):
+    return os.environ.get(name, _SECRETS.get(name, default))
+
+
+CLOUD_API_BASE_URL = _setting("CLOUD_API_BASE_URL", "https://api.text.skin")
+CLOUD_API_TOKEN = _setting("CLOUD_API_TOKEN")
+
 # LLM General Captcha recognition config (supports OpenAI, Qwen, DeepSeek compatible endpoints)
 CAPTCHA_LLM_CONFIG = None
 CAPTCHA_LLM_CONFIG = {
     "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
-    "api_key": "sk-893e895724c6403d81374e515ffaf427",
+    "api_key": _setting("CAPTCHA_LLM_API_KEY"),
     "model": "qwen3-vl-plus",
     "timeout": 30.0,
 }
 
 REDIS_CFG = {
-    "host": "text.skin",
-    "port": 6379,
-    "db": 0,
-    "password": "STEs2x6ML0U1HlpE9SojM6YU7QPhqzY8"
+    "host": _setting("REDIS_HOST", "text.skin"),
+    "port": int(_setting("REDIS_PORT", "6379")),
+    "db": int(_setting("REDIS_DB", "0")),
+    "password": _setting("REDIS_PASSWORD") or None
 }
 
 # Test task: when configured, task rotation directly uses this task

+ 20 - 5
core_types.py

@@ -1,9 +1,10 @@
 import json
 import time
 import socket
+import threading
 from datetime import datetime, timezone
 from dataclasses import dataclass, field, asdict
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, PrivateAttr
 from enum import Enum
 from typing import Optional, List, Any, Dict
 
@@ -155,12 +156,15 @@ class QueryWaitConfig(BaseModel):
     random_min: int = 0
     random_max: int = 0
 
-class PluginConfig(BaseModel):
+class PluginModuleConfig(BaseModel):
     lib_path: str = Field(default="plugins")
     plugin_name: str = Field(default="")
     plugin_bin: str = Field(default="")
     plugin_proto: str = Field(default="IPlugin")
 
+# Semantic alias for backwards compatibility
+PluginBinConfig = PluginModuleConfig
+
 class AppointmentType(BaseModel):
     weight: int = 1
     routing_key: str = ''
@@ -170,6 +174,7 @@ class AppointmentType(BaseModel):
 
 # === Sentinel Config ===
 class SentinelConfig(BaseModel):
+    max_concurrency: int = Field(default=8, ge=1)
     account_source: str = "built-in"
     account_pool_id: str = ""
     target_instances: int = 1
@@ -179,6 +184,8 @@ class SentinelConfig(BaseModel):
 
 # === Booker Config ===
 class BookerConfig(BaseModel):
+    max_concurrency: int = Field(default=8, ge=1)
+    signal_poll_interval: float = Field(default=0.5, ge=0.1)
     account_source: str = "built-in" # "built-in" or "order"
     account_pool_id: str = ""        # Account pool name (used in built-in mode only)
     account_cd: int = 180*60         # Account cooldown in seconds (used in built-in mode only)
@@ -188,6 +195,8 @@ class BookerConfig(BaseModel):
     max_bookings_per_account: int = 1
 
 class GroupConfig(BaseModel):
+    shutdown_timeout: float = Field(default=5.0, ge=0)
+    cleanup_timeout: float = Field(default=3.0, gt=0)
     identifier: str = ""
     debug: bool = False
     enable: bool = False
@@ -201,7 +210,7 @@ class GroupConfig(BaseModel):
     login_backoff: int = 1800           # Cooldown seconds after session creation rate limit
     sentinel: SentinelConfig = Field(default_factory=SentinelConfig)
     booker: BookerConfig = Field(default_factory=BookerConfig)
-    plugin_config: PluginConfig = Field(default_factory=PluginConfig)
+    plugin_config: PluginModuleConfig = Field(default_factory=PluginModuleConfig)
     appointment_types: List[AppointmentType] = Field(default_factory=list)
     website: str = ""
     free_config: Dict[str, Any] = Field(default_factory=dict)
@@ -224,6 +233,8 @@ class Proxy(BaseModel):
     password: str = ""
     
 class BookerStandaloneConfig(BaseModel):
+    shutdown_timeout: float = Field(default=5.0, ge=0)
+    cleanup_timeout: float = Field(default=3.0, gt=0)
     identifier: str = ""
     enable: bool = True
     debug: bool = False
@@ -235,7 +246,7 @@ class BookerStandaloneConfig(BaseModel):
     keep_alive: int = 60                # Keepalive interval in seconds
     query_wait: QueryWaitConfig = Field(default_factory=QueryWaitConfig)
     login_interval: int = 1800          # Login interval in seconds
-    plugin_config: PluginConfig = Field(default_factory=PluginConfig)
+    plugin_config: PluginModuleConfig = Field(default_factory=PluginModuleConfig)
     appointment_types: List[AppointmentType] = Field(default_factory=list)
     user_preferences: Dict[str, Any] = Field(default_factory=dict)
     free_config: Dict[str, Any] = Field(default_factory=dict)
@@ -303,8 +314,12 @@ class BookResult(BaseModel):
 
 # === Internal Task Object ===
 class Task(BaseModel):
+    _operation_lock: Any = PrivateAttr(default_factory=threading.RLock)
+    busy: bool = Field(default=False, exclude=True)
+    removed: bool = Field(default=False, exclude=True)
+    next_health_check: float = 0.0
     instance: Any              
-    qw_cfg: Optional[QueryWaitConfig] = None,
+    qw_cfg: Optional[QueryWaitConfig] = None
     next_run: float = 0.0      
     book_allowed: bool = True
     # Bound Task ID in order mode

+ 3 - 0
docker-compose.yml

@@ -35,6 +35,7 @@ services:
           memory: 512M
 
   visa-sentinel:
+    stop_grace_period: 30s
     build: .
     image: coordinator:latest
     container_name: coordinator-sentinel
@@ -63,6 +64,7 @@ services:
           memory: 2G
 
   visa-booker:
+    stop_grace_period: 30s
     build: .
     image: coordinator:latest
     container_name: coordinator-booker
@@ -91,6 +93,7 @@ services:
           memory: 2G
 
   visa-standalone:
+    stop_grace_period: 30s
     build: .
     image: coordinator:latest
     container_name: coordinator-standalone

+ 45 - 0
docs/scheduler-reliability.md

@@ -0,0 +1,45 @@
+# 调度可靠性与运行配置
+
+订单租约与浏览器保活现在独立运行。工作节点每 10 秒将正在登录、等待号源或预约的订单 deadline 延长到 120 秒后,长时间等待验证码或邮件不会阻止续租。失败的登录保留恢复 deadline;429 使用登录冷却时间,其他失败由 Sweeper 回收。
+
+预约成功先保存到 `data/booking_outbox.sqlite3`,再将成功状态同步到云端。失败会自动重试,节点重启后也会继续;Sweeper 同样读取该日志,待同步成功订单不会重新入队。Redis 中还保存成功标记,供不共享本地磁盘的 Sweeper 识别。
+
+`BOOKING_OUTBOX_PATH` 可以覆盖日志位置。这个文件包含预约结果及支付链接,应放在持久化、受访问控制的目录。Docker Compose 已将 `./data` 挂载给 Booker 和 Sweeper;自定义部署时应保留该挂载。数据库及 SQLite 辅助文件不进入 Git 和镜像。
+
+如果磁盘和 Redis 同时不可写,运行进程会暂存成功结果并继续续租,存储恢复后重试;在两个存储都不可用期间强制终止进程,无法保证恢复这些结果。云端更新接口也应支持幂等写入,以覆盖“请求已成功、响应丢失”这一跨系统故障窗口。
+
+每个浏览器实例的预约、保活和正常清理互斥。不同实例可并发执行;执行器满时保留任务等待下一轮,不堆积线程或任务队列。停止先等待当前操作,超时后进入强制资源清理阶段;只处理本组持有的浏览器 PID、其子进程和代理隧道。
+
+清理失败的实例会保留引用并重试,清理完成前暂停补建浏览器。配置注入失败和启动中途失败也会回滚已取得的资源。重复 start/stop 不会重复启动线程;失败的 stop 可再次调用,热更新不会覆盖尚未清理完成的组。
+
+Python 无法安全强杀任意线程。若第三方调用在关闭浏览器后仍不返回,stop 会在有限等待后报错,将组保留为 FAILED,供后续重试;后台执行线程使用有并发上限的 daemon worker,避免永久阻止进程退出。SIGINT、SIGTERM 和主流程异常均进入统一 finally 清理,并关闭角色 Redis、日志 Redis 和升级订阅连接。
+
+新增可选配置(现有配置可以继续使用):
+
+```yaml
+shutdown_timeout: 5.0  # 正常操作完成的等待秒数,三个角色均支持
+cleanup_timeout: 3.0   # 强制资源清理的等待秒数
+sentinel:
+  max_concurrency: 8
+booker:
+  max_concurrency: 8
+  signal_poll_interval: 0.5
+```
+
+并发上限在组启动时生效,修改后需重启组。`signal_poll_interval` 支持热更新,最小值 0.1 秒。无可预约任务时不读号源信号;其余情况每轮使用一次 MGET。订单心跳每轮每组使用一次 ZADD,Sweeper 使用 SCAN 增量遍历 tracker。
+
+## 服务凭据
+
+源码不再提供服务密钥。优先读取 `CLOUD_API_TOKEN`、`CAPTCHA_LLM_API_KEY`、`REDIS_PASSWORD` 环境变量,也支持 `REDIS_HOST`、`REDIS_PORT`、`REDIS_DB` 和 `CLOUD_API_BASE_URL`。
+
+本地开发可以将 `config/secrets.example.json` 复制为 `config/secrets.json` 后填写。环境变量优先于该文件。`secrets.json` 已被 Git 和 Docker 构建排除;Compose 通过现有 config 挂载读取它。已有工作区凭据已迁移到这个私有文件,权限设置为 0600。
+
+仓库历史中的旧密钥仍应在服务提供方撤销并换新;仅从当前源码移除不能使旧密钥失效。换新后更新环境变量或本地私有文件并重启进程。
+
+## 离线验证
+
+```sh
+venv/bin/python -m unittest test.test_resource_lifecycle test.test_scheduler_reliability test.test_metadata_exception test.test_cloud_config test.test_proxy_manager -v
+```
+
+回归覆盖长预约续租、登录失败恢复、成功结果落盘和重试、重启恢复、存储故障、并发预约与保活互斥、停止竞态、不同号源互不阻塞、批量 Redis 读写和 Sweeper 的成功订单保护。资源测试另覆盖启动失败回滚、失败清理重试、有限停止、Redis 关闭和真实 SIGTERM;进程强制清理使用测试自身创建的子进程验证。外部服务和浏览器均使用替身。

+ 13 - 5
gco_wrapper.py

@@ -75,17 +75,25 @@ class GCOWrapper:
             self._gco.start()  # Call Sentinel or Booker start
         except Exception:
             self._state = State.FAILED
+            if self._gco:
+                try:
+                    self._gco.stop()
+                except Exception as cleanup_err:
+                    self._log(f'Startup rollback incomplete: {cleanup_err}')
             raise
 
     def stop(self):
-        if self._state != State.RUNNING:
+        if self._state not in (State.RUNNING, State.FAILED, State.STOPPED):
             return
-        self._transition(State.RUNNING, State.STOPPING)
+        self._state = State.STOPPING
         try:
             if self._gco:
                 self._gco.stop()  # Call Sentinel or Booker stop
-        finally:
-            self._transition(State.STOPPING, State.STOPPED)
+        except Exception:
+            self._state = State.FAILED
+            raise
+        else:
+            self._state = State.STOPPED
             
     # ================= Hot Reloading Methods =================
     def update_config(self, new_cfg: GroupConfig):
@@ -115,4 +123,4 @@ class GCOWrapper:
             enable=self.m_cfg.enable,
             plugin_name=self.m_cfg.plugin_config.plugin_name,
             wrapper_state=state_name,
-        )
+        )

+ 101 - 67
main.py

@@ -4,11 +4,12 @@ import time
 import json
 import queue
 import argparse
+import signal
 import threading
 from typing import Dict, Any, List
 
 from core_types import GroupConfig, BookerStandaloneConfig, RoleStatus, utc_now_str
-from gco_wrapper import GCOWrapper
+from gco_wrapper import GCOWrapper, State
 from logger_setup import setup_app_logger
 import configure
 from configure import REDIS_CFG
@@ -87,6 +88,7 @@ def reload_wrappers_for_plugin(
                 wrapper.stop()
             except Exception as e:
                 logger.error(f"Error stopping group [{gid}] during reload: {e}")
+                continue
 
             # Clear old module cache to load new code
             sys.modules.pop(plugin_name, None)
@@ -95,8 +97,8 @@ def reload_wrappers_for_plugin(
                 target_cls = gco_classes[gid]
                 new_wrapper = GCOWrapper(target_cls, wrapper.m_cfg, REDIS_CFG)
                 new_wrapper.load()
-                new_wrapper.start()
                 wrappers[gid] = new_wrapper
+                new_wrapper.start()
                 logger.info(f"Group [{gid}] successfully reloaded with updated plugin '{plugin_name}'.")
             except Exception as e:
                 logger.error(f"Failed to restart group [{gid}] after upgrade: {e}")
@@ -153,34 +155,59 @@ def handle_plugin_upgrades(
     return upgraded_any
 
 
-def start_upgrade_listener(redis_conf: Dict, node_id: str, logger):
-    """Starts Redis PubSub thread to listen for instant upgrade commands dispatched from cloud."""
+def start_upgrade_listener(redis_conf: Dict, node_id: str, logger, stop_event):
+    """Cancellable PubSub listener; every reconnect closes the old connection."""
     def _listen():
         import redis
-        while True:
+        while not stop_event.is_set():
+            client = pubsub = None
             try:
-                r = redis.Redis(**redis_conf)
-                pubsub = r.pubsub()
-                channels = ["vs:channel:upgrade", f"vs:channel:upgrade:{node_id}"]
-                pubsub.subscribe(*channels)
-                logger.info(f"Subscribed to upgrade channels: {channels}")
-
-                for msg in pubsub.listen():
+                conf = dict(redis_conf)
+                conf.update(socket_timeout=1, socket_connect_timeout=1)
+                client = redis.Redis(**conf)
+                pubsub = client.pubsub()
+                pubsub.subscribe("vs:channel:upgrade", f"vs:channel:upgrade:{node_id}")
+                while not stop_event.is_set():
+                    msg = pubsub.get_message(timeout=0.5)
                     if msg and msg.get("type") == "message":
                         try:
-                            raw_data = msg.get("data")
-                            if isinstance(raw_data, bytes):
-                                raw_data = raw_data.decode("utf-8")
-                            payload = json.loads(raw_data)
-                            logger.info(f"Received cloud upgrade command: {payload}")
+                            payload = json.loads(msg["data"])
                             upgrade_queue.put(payload)
-                        except Exception as e:
-                            logger.error(f"Error parsing upgrade message: {e}")
-            except Exception as e:
-                logger.warning(f"PubSub connection error: {e}. Reconnecting in 5s...")
-                time.sleep(5)
+                        except Exception as exc:
+                            logger.error(f"Error parsing upgrade message: {exc}")
+            except Exception as exc:
+                if not stop_event.is_set():
+                    logger.warning(f"PubSub connection error: {exc}. Reconnecting in 5s...")
+            finally:
+                for resource in (pubsub, client):
+                    if resource is not None:
+                        try:
+                            resource.close()
+                        except Exception:
+                            pass
+            stop_event.wait(5)
+
+    thread = threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener")
+    thread.start()
+    return thread
+
 
-    threading.Thread(target=_listen, daemon=True, name="Upgrade-Listener").start()
+def shutdown_groups(wrappers, logger):
+    """Stop groups concurrently so shutdown time does not grow with group count."""
+    def stop(gid, wrapper):
+        try:
+            wrapper.stop()
+        except Exception as exc:
+            logger.error(f"Group [{gid}] cleanup pending: {exc}")
+    threads = [threading.Thread(target=stop, args=(gid, wrapper), daemon=True)
+               for gid, wrapper in list(wrappers.items())]
+    for thread in threads:
+        thread.start()
+    timeout = max((w.m_cfg.shutdown_timeout + w.m_cfg.cleanup_timeout + 1
+                   for w in wrappers.values()), default=1)
+    deadline = time.monotonic() + timeout
+    for thread in threads:
+        thread.join(max(0, deadline - time.monotonic()))
 
 
 def sync_groups(
@@ -202,6 +229,7 @@ def sync_groups(
                 wrappers[gid].stop()
             except Exception as e:
                 logger.error(f"Error stopping group [{gid}]: {e}")
+                continue
             del wrappers[gid]
             gco_classes.pop(gid, None)
 
@@ -213,19 +241,20 @@ def sync_groups(
         target_cls = get_gco_class(role, cfg)
 
         if gid in wrappers:
-            if gco_classes.get(gid) != target_cls:
+            if gco_classes.get(gid) != target_cls or wrappers[gid]._state != State.RUNNING:
                 logger.info(f"Mode changed for group [{gid}]. Restarting with {target_cls.__name__}...")
                 try:
                     wrappers[gid].stop()
                 except Exception as e:
                     logger.error(f"Error stopping group [{gid}]: {e}")
+                    continue
                 
                 try:
                     wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
                     wrapper.load()
-                    wrapper.start()
                     wrappers[gid] = wrapper
                     gco_classes[gid] = target_cls
+                    wrapper.start()
                 except Exception as e:
                     logger.error(f"Failed to restart group [{gid}]: {e}")
             else:
@@ -239,9 +268,9 @@ def sync_groups(
             try:
                 wrapper = GCOWrapper(target_cls, cfg, REDIS_CFG)
                 wrapper.load()
-                wrapper.start()
                 wrappers[gid] = wrapper
                 gco_classes[gid] = target_cls
+                wrapper.start()
             except Exception as e:
                 logger.error(f"Failed to start group [{gid}]: {e}")
 
@@ -273,48 +302,52 @@ def main():
 
     redis_client = SafeRedisClient(REDIS_CFG, logger.info)
 
-    # Start real-time cloud command listener (handles upgrade commands)
-    start_upgrade_listener(REDIS_CFG, node_id, logger)
-
-    groups_conf, current_hash, _ = load_remote_or_cache(
-        role=role,
-        node_id=node_id,
-        local_cache_path=config_path,
-        use_cloud=use_cloud,
-        logger=logger.info,
-    )
-
+    stop_event = threading.Event()
+    previous_handlers = {}
+    for signum in (signal.SIGINT, signal.SIGTERM):
+        previous_handlers[signum] = signal.signal(signum, lambda *_: stop_event.set())
     wrappers: Dict[str, GCOWrapper] = {}
     gco_classes: Dict[str, type] = {}
+    listener = None
+    try:
+        # Start real-time cloud command listener (handles upgrade commands)
+        listener = start_upgrade_listener(REDIS_CFG, node_id, logger, stop_event)
 
-    sync_groups(role, groups_conf, wrappers, gco_classes, logger)
-    logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
+        groups_conf, current_hash, _ = load_remote_or_cache(
+            role=role,
+            node_id=node_id,
+            local_cache_path=config_path,
+            use_cloud=use_cloud,
+            logger=logger.info,
+        )
 
-    # Print locally installed plugins and initial RoleStatus summary
-    initial_status = collect_role_status(role, node_id, wrappers)
-    installed_count = len(initial_status.installed_plugins)
-    active_count = len(wrappers)
-    logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
 
-    report_ttl = max(180, int(args.status_interval * 3))
-    try:
-        redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
-    except Exception as e:
-        logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
+        sync_groups(role, groups_conf, wrappers, gco_classes, logger)
+        logger.info(f"Successfully initialized {len(wrappers)} active group(s).")
 
-    if use_cloud:
-        logger.info(f"Cloud config active for role '{role}' (node_id: '{node_id}'). Polling every {args.poll_interval}s.")
+        # Print locally installed plugins and initial RoleStatus summary
+        initial_status = collect_role_status(role, node_id, wrappers)
+        installed_count = len(initial_status.installed_plugins)
+        active_count = len(wrappers)
+        logger.info(f"📊 Initialized RoleStatus: {active_count} active group(s), {installed_count} installed plugin(s).")
 
-    last_poll_time = time.time()
-    last_status_report_time = time.time()
+        report_ttl = max(180, int(args.status_interval * 3))
+        try:
+            redis_client.setex(status_key, report_ttl, json.dumps(initial_status.to_dict()))
+        except Exception as e:
+            logger.warning(f"Error reporting initial RoleStatus to Redis: {e}")
 
-    try:
-        while True:
-            time.sleep(1)
+        if use_cloud:
+            logger.info(f"Cloud config active for role '{role}' (node_id: '{node_id}'). Polling every {args.poll_interval}s.")
+
+        last_poll_time = time.time()
+        last_status_report_time = time.time()
+
+        while not stop_event.wait(1):
             now = time.time()
 
             # 1. Process instant upgrade commands from cloud (PubSub)
-            while not upgrade_queue.empty():
+            while not stop_event.is_set() and not upgrade_queue.empty():
                 try:
                     payload = upgrade_queue.get_nowait()
                     plugins_items = []
@@ -364,17 +397,18 @@ def main():
                 except Exception as e:
                     logger.warning(f"Error polling cloud config for role '{role}' (node_id: '{node_id}'): {e}")
 
-    except KeyboardInterrupt:
-        logger.info(f"Shutting down {role}...")
-        for gid, wrapper in list(wrappers.items()):
+    finally:
+        stop_event.set()
+        shutdown_groups(wrappers, logger)
+        if listener is not None:
+            listener.join(2)
+        for resource in (redis_client, stream_logger):
             try:
-                wrapper.stop()
-            except Exception as e:
-                logger.error(f"Error stopping group [{gid}]: {e}")
-        try:
-            stream_logger.close()
-        except Exception:
-            pass
+                resource.close()
+            except Exception as exc:
+                logger.error(f'Runner connection cleanup failed: {exc}')
+        for signum, handler in previous_handlers.items():
+            signal.signal(signum, handler)
 
 
 if __name__ == "__main__":

+ 23 - 5
main_sweeper.py

@@ -9,6 +9,7 @@ from configure import REDIS_CFG
 from logger_setup import setup_app_logger
 from toolkit.cloud_api import CloudApi
 from core_types import BizException
+from utils.booking_outbox import BookingOutbox
 
 
 class OrphanTaskSweeper:
@@ -21,6 +22,7 @@ class OrphanTaskSweeper:
         safe_conf.setdefault('health_check_interval', 30)
         self.redis_client = redis.Redis(**safe_conf)
         self._running = True
+        self._outbox = BookingOutbox()
 
     def stop(self, *args):
         if self._running:
@@ -49,9 +51,8 @@ class OrphanTaskSweeper:
 
     def _sweep(self):
         # 扫描集群中所有 Worker 的 tracking keys
-        tracker_keys = self.redis_client.keys("vs:worker:tasks_tracker:*")
-        if not tracker_keys:
-            return
+        self._outbox.flush(CloudApi.Instance(), self.redis_client, self.logger.warning)
+        tracker_keys = self.redis_client.scan_iter(match="vs:worker:tasks_tracker:*", count=100)
 
         current_time = time.time()
 
@@ -78,6 +79,19 @@ class OrphanTaskSweeper:
     def _rescue_task(self, tracker_key: str, task_id: int):
         self.logger.warning(f"⚠️ Heartbeat lost for task={task_id} in {tracker_key}. Attempting rescue...")
         try:
+            deadline = self.redis_client.zscore(tracker_key, task_id)
+            if deadline is None or deadline > time.time():
+                return
+            # A successful booking must be synchronized, never rebooked.
+            if self._outbox.get(task_id) is not None:
+                return
+            success = self.redis_client.get(self._outbox.marker(task_id))
+            if success:
+                import json
+                CloudApi.Instance().update_vas_task(str(task_id), json.loads(success))
+                self.redis_client.zrem(tracker_key, task_id)
+                self.redis_client.delete(self._outbox.marker(task_id))
+                return
             task_data = None
             try:
                 task_data = CloudApi.Instance().get_vas_task(str(task_id))
@@ -94,13 +108,17 @@ class OrphanTaskSweeper:
                 return
 
             current_status = task_data.get('status', '')
-            if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed']:
+            if current_status in ['cancelled', 'pause', 'grabbed', 'success', 'completed', 'failed']:
                 self.logger.info(f"Task={task_id} status '{current_status}'. Removing from tracker.")
                 self.redis_client.zrem(tracker_key, task_id)
                 return
 
             # 云端状态还是 processing,证明 Worker 真的死机了,强制归还
             self.logger.info(f"Returning dead task={task_id} to queue.")
+            # Cloud reads can take seconds; a worker may have renewed meanwhile.
+            deadline = self.redis_client.zscore(tracker_key, task_id)
+            if deadline is None or deadline > time.time():
+                return
             CloudApi.Instance().return_vas_task_to_queue(task_id)
             
             self.redis_client.zrem(tracker_key, task_id)
@@ -125,4 +143,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

+ 113 - 104
sentinel.py

@@ -1,4 +1,5 @@
 import os
+from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
 import time
 import json
 import random
@@ -17,6 +18,7 @@ from toolkit.cloud_api import CloudApi
 from toolkit.proxy_manager import ProxyManager
 from toolkit.notifier import notify_date_changes
 from utils.safe_redis_cli import SafeRedisClient
+from toolkit.task_executor import TaskExecutor
 
 class SentinelGCO:
     def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
@@ -27,9 +29,12 @@ class SentinelGCO:
         self.m_lock = threading.RLock()
         self.m_stop_event = threading.Event()
         self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
+        init_lifecycle(self)
         self.m_last_group_query_time = 0.0
         self.m_spawn_cooldown_until = 0.0
         self.m_current_dates = {}
+        self._executor = TaskExecutor(cfg.sentinel.max_concurrency, 'Sentinel')
+        self._background_threads = []
 
     def _log(self, message):
         if self.m_logger:
@@ -88,6 +93,7 @@ class SentinelGCO:
             updated_at=utc_now_str(),
         )
 
+    @start_once
     def start(self):
         if not self.m_cfg.enable:
             return
@@ -96,34 +102,48 @@ class SentinelGCO:
         class_name = "".join(part.title() for part in plugin_name.split('_'))
         plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
         self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
-        threading.Thread(target=self._monitor_loop, daemon=True, name="Sentinel-Monitor").start()
-        threading.Thread(target=self._creator_loop, daemon=True, name="Sentinel-Creator").start()
+        for target in (self._monitor_loop, self._creator_loop):
+            thread = threading.Thread(target=target, daemon=True)
+            self._background_threads.append(thread)
+            thread.start()
 
+    @serialized_stop
     def stop(self):
         self._log("Stopping Sentinel...")
         self.m_stop_event.set()
+        deadline = time.monotonic() + self.m_cfg.shutdown_timeout
+        self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
+        join_threads(self._background_threads, max(0, deadline - time.monotonic()))
+        with self.m_lock:
+            for task in self.m_tasks:
+                task.removed = True
+                self._resources.track(task.instance)
+        self._resources.close(self.m_cfg.cleanup_timeout)
+        threads_done = self._executor.shutdown(timeout=0.1) and join_threads(self._background_threads, 0.1)
+        if not self._resources.empty or not threads_done:
+            raise RuntimeError('Sentinel shutdown incomplete; resources retained for retry')
         with self.m_lock:
-            tasks_to_cleanup = list(self.m_tasks)
             self.m_tasks.clear()
-        for task in tasks_to_cleanup:
-            self._cleanup_task(task, "sentinel stopped")
+        self.redis_client.close()
 
     def _cleanup_instance(self, instance, reason: str = ""):
-        try:
-            if instance:
-                self._log(f"🧹 Cleaned up instance. Reason: {reason}")
-                instance.cleanup()
-        except Exception as e:
-            self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
+        if instance is None:
+            return True
+        cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
+        if not cleaned:
+            self._log(f"Cleanup retained for retry ({reason})")
+        return cleaned
 
     def _cleanup_task(self, task: Task, reason: str):
         if task and task.instance:
-            self._cleanup_instance(task.instance, reason)
+            with task._operation_lock:
+                self._cleanup_instance(task.instance, reason)
 
     def _remove_task(self, task: Task, reason: str):
         removed = False
         with self.m_lock:
             if task in self.m_tasks:
+                task.removed = True
                 self.m_tasks.remove(task)
                 removed = True
         if removed:
@@ -295,97 +315,81 @@ class SentinelGCO:
         else:
             return current_utc_time >= start_time or current_utc_time <= end_time
 
-    def _monitor_loop(self):
-        self._log("Monitor loop started.")
-        
-        self.m_last_group_query_time = 0.0 
-        while not self.m_stop_event.is_set():
-            try:
-                time.sleep(1)
+    def _monitor_task(self, task, apt_type):
+        try:
+            with task._operation_lock, self._resources.operation(task.instance):
+                if self.m_stop_event.is_set() or task.removed:
+                    return
                 now = time.time()
-                
+                if now >= task.next_health_check:
+                    if not task.instance.health_check():
+                        self._remove_task(task, "health check failed")
+                        return
+                    task.next_health_check = now + 10
+                if apt_type is None:
+                    return
+                qw = self.m_cfg.sentinel.query_wait
+                if qw.mode == QueryWaitMode.Loop:
+                    interval = 1
+                elif qw.mode == QueryWaitMode.Fixed:
+                    interval = qw.fixed_wait
+                else:
+                    interval = random.uniform(qw.random_min, qw.random_max)
+                result = task.instance.query(apt_type)
+                result.apt_type = apt_type
                 with self.m_lock:
-                    tasks_to_check = list(self.m_tasks)
-
-                active_tasks = []
-                dead_tasks = []
-                for t in tasks_to_check:
-                    if t.instance.health_check():
-                        active_tasks.append(t)
-                    else:
-                        dead_tasks.append(t)
-          
-                if dead_tasks:
-                    with self.m_lock:
-                        for t in dead_tasks:
-                            if t in self.m_tasks:
-                                self.m_tasks.remove(t)
-                    for t in dead_tasks:
-                        self._cleanup_task(t, "health check failed")
-                
-                if not active_tasks:
+                    notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
+                if result.success:
+                    payload = {"group_id": self.m_cfg.identifier,
+                               "apt_type": apt_type.model_dump(),
+                               "query_result": result.to_snapshot_payload(),
+                               "timestamp": time.time()}
+                    self.redis_client.setex(self._get_redis_key(apt_type.routing_key),
+                                            self.m_cfg.sentinel.signal_ttl, json.dumps(payload))
+                task.next_run = time.time() + max(0.05, interval)
+                if task.instance.config.proxy.port > 0:
+                    ProxyManager.Instance().mark_proxy_result(
+                        proxy=task.instance.config.proxy.model_dump(),
+                        plugin_name=self.m_cfg.plugin_config.plugin_name,
+                        action="query", success=True, http_code=200)
+        except Exception as exc:
+            self._dispatch_error_policy("query", exc, task=task)
+        finally:
+            with self.m_lock:
+                task.busy = False
+
+    def _schedule_monitor_jobs(self):
+        with self.m_lock:
+            tasks = sorted(self.m_tasks, key=lambda t: t.next_run)
+            gap = max(0.05, self._get_average_interval() / max(1, len(tasks)))
+            for task in tasks:
+                if task.busy or task.removed or self.m_stop_event.is_set():
+                    continue
+                now = time.time()
+                apt_type = None
+                apt_types = self.m_cfg.appointment_types
+                if (apt_types and now >= task.next_run
+                        and now - self.m_last_group_query_time >= gap):
+                    weights = [max(0, apt.weight) for apt in apt_types]
+                    apt_type = (random.choices(apt_types, weights=weights, k=1)[0]
+                                if sum(weights) else random.choice(apt_types))
+                if apt_type is None and now < task.next_health_check:
                     continue
+                task.busy = True
+                if self._executor.submit(self._monitor_task, task, apt_type):
+                    if apt_type is not None:
+                        self.m_last_group_query_time = now
+                else:
+                    task.busy = False
 
-                avg_interval = self._get_average_interval()
-                global_gap = max(1.0, avg_interval / len(active_tasks)) 
-                
-                active_tasks.sort(key=lambda x: x.next_run)
-                for task in active_tasks:
-                    if now < task.next_run:
-                        continue
-                
-                    if now - self.m_last_group_query_time < global_gap:
-                        break
-
-                    apt_types = self.m_cfg.appointment_types
-                    if not apt_types:
-                        continue
-                    weights = [max(0.0, float(item.weight)) for item in apt_types]
-                    if sum(weights) > 0:
-                        apt_type = random.choices(apt_types, weights=weights, k=1)[0]
-                    else:
-                        apt_type = random.choice(apt_types)
-                    
-                    interval = 30
-                    mode = task.qw_cfg.mode
-                    if mode == QueryWaitMode.Loop:
-                        interval = 1
-                    elif mode == QueryWaitMode.Fixed:
-                        interval = task.qw_cfg.fixed_wait
-                    elif mode == QueryWaitMode.Random:
-                        interval = random.randint(task.qw_cfg.random_min, task.qw_cfg.random_max)
-
-                    self.m_last_group_query_time = now
-                    try:
-                        result = task.instance.query(apt_type)
-                        result.apt_type = apt_type
-                        notify_date_changes(result, cache=self.m_current_dates, logger=self._log)
-                        if result.success:
-                            ttl = self.m_cfg.sentinel.signal_ttl
-                            self._log(f"🔥 SLOT FOUND! Writing signal to Redis (TTL: {ttl}s)")
-                            payload = {
-                                "group_id": self.m_cfg.identifier,
-                                "apt_type": apt_type.model_dump(),
-                                "query_result": result.to_snapshot_payload(),
-                                "timestamp": time.time()
-                            }
-                            redis_key = self._get_redis_key(apt_type.routing_key)
-                            self.redis_client.setex(redis_key, ttl, json.dumps(payload))
-                        task.next_run = time.time() + interval
-                        # Record successful query metrics for proxy
-                        if task.instance.config.proxy.port > 0:
-                            ProxyManager.Instance().mark_proxy_result(
-                                proxy=task.instance.config.proxy.model_dump(),
-                                plugin_name=self.m_cfg.plugin_config.plugin_name,
-                                action="query",
-                                success=True,
-                                http_code=200
-                            )
-                    except Exception as e:
-                        self._dispatch_error_policy("query", e, task=task)
-                    break
-            except Exception as e:
-                self._dispatch_error_policy("monitor_loop", e)
+    def _monitor_loop(self):
+        self._log("Monitor loop started.")
+        while not self.m_stop_event.wait(0.05):
+            try:
+                if self.m_cfg.enable and self._is_within_active_hours():
+                    self._schedule_monitor_jobs()
+            except Exception as exc:
+                self._dispatch_error_policy("monitor_loop", exc)
 
     def _creator_loop(self):
         self._log("Creator loop started.")
@@ -404,6 +408,8 @@ class SentinelGCO:
                 self._spawn_sentinel_worker()
 
     def _spawn_sentinel_worker(self):
+        if self.m_stop_event.is_set() or not self._resources.retry_failed():
+            return
         instance = None
         task = None
         success = False
@@ -430,13 +436,16 @@ class SentinelGCO:
                 plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
 
             instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
-            instance.set_log(self.m_logger)
-            instance.set_config(plg_cfg)
-            
-            instance.create_session()
+            self._resources.track(instance)
+            with self._resources.operation(instance):
+                instance.set_log(self.m_logger)
+                instance.set_config(plg_cfg)
+                instance.create_session()
             
             task = Task(instance=instance, qw_cfg=self.m_cfg.sentinel.query_wait, next_run=time.time(), book_allowed=False)
             with self.m_lock:
+                if self.m_stop_event.is_set():
+                    return
                 self.m_tasks.append(task)
             success = True
             if plg_cfg.proxy.port > 0:
@@ -452,4 +461,4 @@ class SentinelGCO:
             self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
         finally:
             if not success and instance:
-                self._cleanup_instance(instance, "spawn failed")
+                self._cleanup_instance(instance, "spawn failed")

+ 58 - 22
standalone.py

@@ -1,4 +1,5 @@
 import os
+from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
 import time
 import random
 import threading
@@ -53,7 +54,9 @@ class StandaloneGCO:
         self.m_next_keepalive_time = 0
         self.m_last_wait_log_time = 0.0
         self.m_current_dates = {}
+        self._worker_thread = None
         self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
+        init_lifecycle(self)
 
     def _log(self, message: str):
         prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
@@ -125,6 +128,7 @@ class StandaloneGCO:
             updated_at=utc_now_str(),
         )
 
+    @start_once
     def start(self):
         """Non-blocking start: registers plugin and starts background worker thread."""
         if not self.m_cfg.enable:
@@ -137,28 +141,45 @@ class StandaloneGCO:
         self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
 
         worker_name = f"Standalone-{self.m_cfg.identifier}"
-        threading.Thread(target=self._worker_loop, daemon=True, name=worker_name).start()
+        self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True, name=worker_name)
+        self._worker_thread.start()
 
+    @serialized_stop
     def stop(self):
         """Gracefully stop background worker and clean up resources."""
         self._log("Stopping Standalone GCO...")
         self.m_stop_event.set()
-        self._cleanup_instance("standalone stopped")
+        threads = [self._worker_thread] if self._worker_thread else []
+        join_threads(threads, self.m_cfg.shutdown_timeout)
+        if self.m_instance is not None:
+            self._resources.track(self.m_instance)
+        self._resources.close(self.m_cfg.cleanup_timeout)
+        threads_done = join_threads(threads, 0.1)
+        if not self._resources.empty or not threads_done:
+            raise RuntimeError('Standalone shutdown incomplete; resources retained for retry')
+        self.m_instance = None
+        self.redis_client.close()
 
     def _cleanup_instance(self, reason: str = ""):
         with self.m_lock:
-            if self.m_instance is not None:
-                try:
-                    self._log(f"🧹 Cleaned up instance. Reason: {reason}")
-                    self.m_instance.cleanup()
-                except Exception as e:
-                    self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
-                finally:
+            instance = self.m_instance
+        if instance is None:
+            return True
+        cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
+        if cleaned:
+            with self.m_lock:
+                if self.m_instance is instance:
                     self.m_instance = None
+        else:
+            self._log(f"Cleanup retained for retry ({reason})")
+        return cleaned
 
     def _init_instance(self):
         """Initialize single instance and log in to create a session."""
-        self._cleanup_instance("recreating new session")
+        if not self._resources.retry_failed():
+            raise RuntimeError('Previous instance cleanup is still pending')
+        if not self._cleanup_instance("recreating new session"):
+            raise RuntimeError('Previous instance cleanup is still pending')
 
         now = time.time()
         if now < self.m_last_login_time + self.m_cfg.login_interval:
@@ -177,17 +198,25 @@ class StandaloneGCO:
         plg_cfg.session_max_life = self.m_cfg.session_max_life
 
         instance = self.m_factory.create(self.m_cfg.identifier, plugin_name)
-        instance.set_log(self.m_logger)
-        instance.set_config(plg_cfg)
-        with self.m_lock:
-            self.m_instance = instance
-
-        self._log("Creating session...")
+        self._resources.track(instance)
         try:
-            instance.create_session()
+            with self._resources.operation(instance):
+                instance.set_log(self.m_logger)
+                instance.set_config(plg_cfg)
+                with self.m_lock:
+                    if self.m_stop_event.is_set():
+                        return
+                    self.m_instance = instance
+                instance.create_session()
         except Exception:
-            self._cleanup_instance("create_session failed")
+            if self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout):
+                with self.m_lock:
+                    if self.m_instance is instance:
+                        self.m_instance = None
             raise
+        finally:
+            if self.m_stop_event.is_set():
+                self._resources.cleanup(instance, force=True, timeout=self.m_cfg.cleanup_timeout)
         self.m_last_login_time = time.time()
         interval = self._get_wait_interval()
         self.m_next_query_time = time.time() + interval
@@ -320,6 +349,8 @@ class StandaloneGCO:
                 if self.m_instance is None:
                     try:
                         self._init_instance()
+                        if self.m_stop_event.is_set():
+                            break
                     except Exception as e:
                         self._dispatch_error_policy("create_session", e)
                         continue
@@ -339,7 +370,8 @@ class StandaloneGCO:
 
                     # Periodic keepalive
                     if self.m_instance and now >= self.m_next_keepalive_time:
-                        self.m_instance.keep_alive()
+                        with self._resources.operation(self.m_instance):
+                            self.m_instance.keep_alive()
                         self.m_next_keepalive_time = now + keep_alive_sec
                         self._log(f"Session keepalive sent. Next query at {next_query_str} (remaining: {remaining:.0f}s)...")
                         self.m_last_wait_log_time = now
@@ -367,7 +399,8 @@ class StandaloneGCO:
 
                 self._log(f"Querying slots for {apt_type.routing_key}...")
                 try:
-                    query_result = self.m_instance.query(apt_type)
+                    with self._resources.operation(self.m_instance):
+                        query_result = self.m_instance.query(apt_type)
                     query_result.apt_type = apt_type
                     notify_date_changes(query_result, cache=self.m_current_dates, logger=self._log)
                 except Exception as e:
@@ -378,7 +411,10 @@ class StandaloneGCO:
                 if query_result.success:
                     self._log("🔥 SLOT FOUND! Initiating AUTO-BOOKING...")
                     try:
-                        book_result = self.m_instance.book(query_result, self.m_cfg.user_preferences)
+                        if self.m_stop_event.is_set():
+                            break
+                        with self._resources.operation(self.m_instance):
+                            book_result = self.m_instance.book(query_result, self.m_cfg.user_preferences)
                         notify_book_result(book_result, logger=self._log)
                         if book_result.success:
                             self._log(f"🎉 BOOKING SUCCESSFUL for {apt_type.routing_key}!")
@@ -396,4 +432,4 @@ class StandaloneGCO:
                 self._log(f"Next query scheduled in {interval:.1f}s (at {datetime.fromtimestamp(self.m_next_query_time).strftime('%H:%M:%S')})...")
 
             except Exception as e:
-                self._dispatch_error_policy("worker_loop", e)
+                self._dispatch_error_policy("worker_loop", e)

+ 6 - 4
test/test.py

@@ -1,7 +1,9 @@
 import cv2
 import numpy as np
 
-img = cv2.imread('image.png')
-# 使用中值滤波去噪,核大小为3
-dst = cv2.medianBlur(img, 3)
-cv2.imwrite('output.png', dst)
+if __name__ == '__main__':
+    img = cv2.imread('image.png')
+    if img is not None:
+        # 使用中值滤波去噪,核大小为3
+        dst = cv2.medianBlur(img, 3)
+        cv2.imwrite('output.png', dst)

+ 9 - 12
test/test_publish_slot.py

@@ -1,14 +1,7 @@
 import json
 import redis
 
-# Redis 连接
-r = redis.Redis(
-    host="text.skin",
-    port=6379,
-    db=0,
-    password="STEs2x6ML0U1HlpE9SojM6YU7QPhqzY8",
-    decode_responses=True
-)
+from configure import REDIS_CFG
 
 # 使用的键名(原 Channel 名称)
 key = "vs:signal:slot.lon.fr.tourist"
@@ -47,8 +40,12 @@ message = {
     "timestamp": 1777745853
 }
 
-# 写入 Redis 键,有效期 30 秒
-r.setex(key, 30, json.dumps(message, ensure_ascii=False))
+def main():
+    r = redis.Redis(**REDIS_CFG, decode_responses=True)
+    r.setex(key, 30, json.dumps(message, ensure_ascii=False))
+    print(f"写入成功 -> 键: {key}")
+    print("有效期: 30 秒")
 
-print(f"写入成功 -> 键: {key}")
-print(f"有效期: 30 秒")
+
+if __name__ == "__main__":
+    main()

+ 290 - 0
test/test_resource_lifecycle.py

@@ -0,0 +1,290 @@
+"""Offline resource regressions, including a real SIGTERM and an owned child PID."""
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from booker import BuiltinBookerGCO
+from core_types import BookerStandaloneConfig, GroupConfig
+from gco_wrapper import GCOWrapper, State
+from sentinel import SentinelGCO
+from standalone import StandaloneGCO
+from toolkit.task_executor import TaskExecutor
+from utils.plugin_utils import cleanup_browser_workspace
+from utils.resource_lifecycle import InstanceRegistry
+from utils.safe_redis_cli import SafeRedisClient
+
+
+class ResourceLifecycleTests(unittest.TestCase):
+    def test_configuration_failure_removes_partially_initialized_workspace(self):
+        with tempfile.TemporaryDirectory() as root:
+            workspace = Path(root) / 'profile'
+            workspace.mkdir()
+            plugin = MagicMock()
+            plugin.set_config.side_effect = ValueError('invalid config')
+            plugin.cleanup.side_effect = lambda: shutil.rmtree(workspace)
+            g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
+            g.m_factory.create = MagicMock(return_value=plugin)
+            with patch('standalone.ProxyManager.Instance') as manager:
+                manager.return_value.get_next_proxy.return_value = {}
+                with self.assertRaises(ValueError):
+                    g._init_instance()
+            self.assertFalse(workspace.exists())
+            self.assertFalse(g._resources._entries)
+            g.stop()
+
+    def test_failed_cleanup_keeps_handle_and_retry_can_release_it(self):
+        g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
+        plugin = SimpleNamespace(page=MagicMock(), tunnel=None, root_workspace=None)
+        attempts = []
+        def cleanup():
+            attempts.append(1)
+            if len(attempts) == 1:
+                # Even a plugin that drops its page prematurely can be recovered.
+                plugin.page = None
+                raise RuntimeError('cleanup failed')
+        plugin.cleanup = cleanup
+        page = plugin.page
+        g.m_instance = plugin
+        self.assertFalse(g._cleanup_instance())
+        self.assertIs(g.m_instance, plugin)
+        self.assertTrue(g._resources._entries)
+        g.stop()
+        page.quit.assert_called()
+        self.assertIsNone(g.m_instance)
+        self.assertFalse(g._resources._entries)
+
+    def test_browser_cleanup_failure_is_reported_and_tunnel_is_still_stopped(self):
+        page = SimpleNamespace(process_id=None, quit=MagicMock(side_effect=RuntimeError('CDP failed')))
+        tunnel = MagicMock()
+        with self.assertRaisesRegex(RuntimeError, 'retaining its handle'):
+            cleanup_browser_workspace(page=page, tunnel=tunnel)
+        self.assertEqual(page.quit.call_count, 2)
+        tunnel.stop.assert_called_once()
+
+    def test_force_fallback_terminates_only_owned_child(self):
+        child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])
+        try:
+            page = SimpleNamespace(process_id=child.pid,
+                                   quit=MagicMock(side_effect=RuntimeError('CDP failed')))
+            cleanup_browser_workspace(page=page)
+            child.wait(timeout=2)
+            self.assertIsNotNone(child.returncode)
+        finally:
+            if child.poll() is None:
+                child.kill()
+                child.wait()
+
+    def test_partial_start_is_rolled_back_and_failed_wrapper_can_stop(self):
+        class PartialStart(BuiltinBookerGCO):
+            def _booking_trigger_loop(self): self.m_stop_event.wait()
+            def _creator_loop(self): self.m_stop_event.wait()
+            def _maintain_loop(self): self.m_stop_event.wait()
+            def _heartbeat_loop(self): self.m_stop_event.wait()
+            def _success_sync_loop(self): self.m_stop_event.wait()
+            def _start_additional_threads(self): raise RuntimeError('partial startup')
+        wrapper = GCOWrapper(PartialStart, GroupConfig(enable=True), {})
+        wrapper.load()
+        with self.assertRaisesRegex(RuntimeError, 'partial startup'):
+            wrapper.start()
+        self.assertEqual(wrapper._state, State.FAILED)
+        self.assertTrue(wrapper._gco.m_stop_event.is_set())
+        self.assertFalse(any(t.is_alive() for t in wrapper._gco._background_threads))
+        wrapper.stop()
+        self.assertEqual(wrapper._state, State.STOPPED)
+
+    def test_blocked_worker_gets_bounded_stop_and_resource_cancellation(self):
+        g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.02, cleanup_timeout=0.1), {}, logger=lambda _: None)
+        g.redis_client = MagicMock()
+        release = threading.Event()
+        entered = threading.Event()
+        plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None,
+                                 cleanup=lambda: release.set())
+        def run():
+            with g._resources.operation(plugin):
+                entered.set()
+                release.wait(2)
+        g._executor.submit(run)
+        self.assertTrue(entered.wait(1))
+        started = time.monotonic()
+        try:
+            try:
+                g.stop()
+            except RuntimeError:
+                # An in-flight cleanup can complete immediately after the deadline.
+                pass
+            self.assertLess(time.monotonic() - started, 0.6)
+            self.assertTrue(release.is_set())
+        finally:
+            release.set()
+            g._executor.shutdown(timeout=1)
+            g.stop()
+        g.redis_client.close.assert_called_once()
+
+    def test_executor_does_not_keep_process_alive_for_a_blocked_call(self):
+        executor = TaskExecutor(1, 'resource-test')
+        release = threading.Event()
+        entered = threading.Event()
+        executor.submit(lambda: (entered.set(), release.wait(2)))
+        self.assertTrue(entered.wait(1))
+        try:
+            self.assertFalse(executor.shutdown(timeout=0.01))
+            self.assertTrue(all(t.daemon for t in executor._threads))
+            self.assertFalse(executor.submit(lambda: None))
+        finally:
+            release.set()
+            self.assertTrue(executor.shutdown(timeout=1))
+
+    def test_redis_is_not_closed_under_an_inflight_heartbeat(self):
+        g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.01, cleanup_timeout=0.05), {}, logger=lambda _: None)
+        g.redis_client = MagicMock()
+        release = threading.Event()
+        entered = threading.Event()
+        thread = threading.Thread(target=lambda: (entered.set(), release.wait(2)),
+                                  name='_heartbeat_loop', daemon=True)
+        g._background_threads.append(thread)
+        thread.start()
+        self.assertTrue(entered.wait(1))
+        try:
+            with self.assertRaises(RuntimeError):
+                g.stop()
+            g.redis_client.close.assert_not_called()
+        finally:
+            release.set()
+            thread.join(1)
+            g.stop()
+        g.redis_client.close.assert_called_once()
+
+    def test_failed_stop_is_retryable_without_overwriting_group(self):
+        plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None,
+                                 cleanup=MagicMock(side_effect=[RuntimeError('busy'), None]))
+        g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
+        g.redis_client = MagicMock()
+        g.m_instance = plugin
+        with self.assertRaises(RuntimeError):
+            g.stop()
+        self.assertIs(g.m_instance, plugin)
+        g.redis_client.close.assert_not_called()
+        g.stop()
+        self.assertIsNone(g.m_instance)
+        g.redis_client.close.assert_called_once()
+
+    def test_stop_closes_redis_in_all_three_roles(self):
+        for cls, cfg in ((StandaloneGCO, BookerStandaloneConfig()),
+                         (BuiltinBookerGCO, GroupConfig()), (SentinelGCO, GroupConfig())):
+            with self.subTest(role=cls.__name__):
+                g = cls(cfg, {}, logger=lambda _: None)
+                raw = MagicMock()
+                g.redis_client._client = raw
+                g.stop()
+                g.stop()
+                raw.close.assert_called_once()
+                raw.connection_pool.disconnect.assert_called_once()
+                g.redis_client.get('after-close')
+                raw.get.assert_not_called()
+
+    def test_diagnostic_retention_does_not_defeat_explicit_stop(self):
+        registry = InstanceRegistry(lambda _: None)
+        plugin = SimpleNamespace(keep_browser_on_error=True, page=MagicMock(),
+                                 tunnel=None, root_workspace=None, cleanup=MagicMock())
+        self.assertFalse(registry.cleanup(plugin))
+        self.assertTrue(registry.close())
+        self.assertFalse(plugin.keep_browser_on_error)
+
+    def test_main_retains_failed_wrapper_for_cleanup_retry(self):
+        import main as app
+        class HalfStarted:
+            def __init__(self, *args, **kwargs): pass
+            def start(self): raise RuntimeError('startup failed')
+            def stop(self): raise RuntimeError('cleanup pending')
+        wrappers, classes = {}, {}
+        config = {'identifier': 'retained', 'enable': True}
+        with patch.object(app, 'get_gco_class', return_value=HalfStarted):
+            app.sync_groups('sentinel', config, wrappers, classes, MagicMock())
+        self.assertIn('retained', wrappers)
+        self.assertEqual(wrappers['retained']._state, State.FAILED)
+        self.assertIsInstance(wrappers['retained']._gco, HalfStarted)
+
+    def test_upgrade_listener_closes_pubsub_and_client_on_stop(self):
+        from main import start_upgrade_listener
+        stop = threading.Event()
+        client = MagicMock()
+        pubsub = client.pubsub.return_value
+        pubsub.get_message.side_effect = lambda **_: (stop.set(), None)[1]
+        with patch('redis.Redis', return_value=client):
+            thread = start_upgrade_listener({}, 'test', MagicMock(), stop)
+            thread.join(1)
+        self.assertFalse(thread.is_alive())
+        pubsub.close.assert_called_once()
+        client.close.assert_called_once()
+
+    def test_stream_logger_closes_worker_and_pool(self):
+        from utils.redis_stream_logger import RedisStreamLogger
+        logger = RedisStreamLogger({}, 'test', 'test')
+        raw = MagicMock()
+        logger._redis_client = raw
+        logger.close()
+        self.assertFalse(logger._worker_thread.is_alive())
+        raw.close.assert_called_once()
+        raw.connection_pool.disconnect.assert_called_once()
+
+    @unittest.skipUnless(os.name == 'posix', 'POSIX signal integration')
+    def test_sigterm_runs_main_finally_and_closes_connections(self):
+        with tempfile.TemporaryDirectory() as root:
+            ready = Path(root) / 'ready'
+            done = Path(root) / 'done'
+            script = '''
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock
+import main as app
+from core_types import GroupConfig, RoleStatus
+ready, done = map(Path, sys.argv[1:])
+sys.argv = ['main.py', 'sentinel', '-c', 'unused.yaml']
+logger = MagicMock()
+redis = MagicMock()
+stream = MagicMock()
+wrapper = MagicMock(m_cfg=GroupConfig())
+wrapper.stop.side_effect = lambda: done.write_text('stopped')
+app.setup_app_logger = lambda *a, **k: logger
+app.SafeRedisClient = lambda *a, **k: redis
+app.RedisStreamLogger.initialize = lambda **k: stream
+app.start_upgrade_listener = lambda *a: None
+app.load_remote_or_cache = lambda **k: ([], '', False)
+app.collect_role_status = lambda *a: RoleStatus()
+def sync(role, config, wrappers, classes, logger):
+    wrappers['test'] = wrapper
+    ready.write_text('ready')
+app.sync_groups = sync
+app.main()
+assert redis.close.call_count == 1
+assert stream.close.call_count == 1
+'''
+            child = subprocess.Popen([sys.executable, '-c', script, str(ready), str(done)],
+                                     cwd=str(Path(__file__).resolve().parent.parent),
+                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+            try:
+                deadline = time.monotonic() + 3
+                while not ready.exists() and child.poll() is None and time.monotonic() < deadline:
+                    time.sleep(0.01)
+                self.assertTrue(ready.exists())
+                child.send_signal(signal.SIGTERM)
+                stdout, stderr = child.communicate(timeout=5)
+                self.assertEqual(child.returncode, 0, stdout + stderr)
+                self.assertTrue(done.exists())
+            finally:
+                if child.poll() is None:
+                    child.kill()
+                    child.communicate()
+
+
+if __name__ == '__main__':
+    unittest.main()

+ 340 - 0
test/test_scheduler_reliability.py

@@ -0,0 +1,340 @@
+"""Offline regressions for order leases, durable results and browser scheduling."""
+import json
+import tempfile
+import threading
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from booker import BuiltinBookerGCO, OrderBookerGCO
+from sentinel import SentinelGCO
+from core_types import (AppointmentType, AvailabilityStatus, BookResult, GroupConfig,
+                        AccountIncompleteError, PluginConfig, QueryResult, RateLimiteddError, Task)
+from main_sweeper import OrphanTaskSweeper
+from toolkit.task_executor import TaskExecutor
+from utils.booking_outbox import BookingOutbox
+from utils.safe_redis_cli import SafeRedisClient
+
+
+class SchedulerTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.cloud = MagicMock()
+        self.cloud_patch = patch('toolkit.cloud_api.CloudApi.Instance', return_value=self.cloud)
+        self.cloud_patch.start()
+        self.addCleanup(self.cloud_patch.stop)
+
+    def make(self, cls=BuiltinBookerGCO):
+        g = cls(GroupConfig(identifier='audit', enable=True), {}, logger=lambda _: None)
+        g.redis_client = MagicMock()
+        if hasattr(g, '_outbox'):
+            g._outbox = BookingOutbox(str(Path(self.tmp.name) / 'outbox.sqlite3'))
+        self.addCleanup(g.stop)
+        return g
+
+    def signal(self, key):
+        return json.dumps({'apt_type': {'routing_key': key}, 'query_result': {}})
+
+    def test_long_builtin_booking_renews_order_lease_without_browser_ping(self):
+        g = self.make()
+        plugin = MagicMock(config=PluginConfig())
+        task = Task(instance=plugin)
+        self.cloud.get_vas_task_pop.return_value = {'id': 42}
+        clock = [1000.0]
+
+        def book(*_):
+            clock[0] += 131
+            g._renew_leases()
+            mapping = g.redis_client.bulk_zadd.call_args.args[1]
+            self.assertEqual(mapping['42'], 1251.0)
+            plugin.keep_alive.assert_not_called()
+            return BookResult()
+
+        plugin.book.side_effect = book
+        with patch('booker.time.time', side_effect=lambda: clock[0]):
+            g._execute_book_job(task, QueryResult())
+        self.assertNotIn('42', g._leased_orders)
+
+    def test_spawn_rate_limit_retains_recovery_deadline(self):
+        g = self.make(OrderBookerGCO)
+        self.cloud.get_vas_task_pop.return_value = {'id': 42, 'user_inputs': {'username': 'audit'}}
+        plugin = MagicMock()
+        plugin.create_session.side_effect = RateLimiteddError()
+        g.m_factory.create = MagicMock(return_value=plugin)
+        g._spawn_worker('a')
+        deadline = g.redis_client.zadd.call_args.args[1]['42']
+        self.assertGreaterEqual(deadline, g.m_spawn_cooldown_until)
+        g.redis_client.zrem.assert_not_called()
+        self.assertNotIn('42', g._leased_orders)
+        plugin.cleanup.assert_called_once()
+
+    def test_failed_success_sync_is_durable_and_retryable(self):
+        g = self.make(OrderBookerGCO)
+        self.cloud.update_vas_task.side_effect = RuntimeError('offline')
+        with patch('booker.notify_book_result'):
+            g._push_success_notification(42, 'order', BookResult(success=True, urn='confirmed'))
+        g.redis_client._client.zrem.assert_not_called()
+        reopened = BookingOutbox(g._outbox.path)
+        self.assertEqual(reopened.get(42)['grabbed_history']['urn'], 'confirmed')
+        self.cloud.update_vas_task.side_effect = None
+        with patch('utils.booking_outbox.time.time', return_value=time.time() + 10):
+            reopened.flush(self.cloud, g.redis_client._client, lambda _: None)
+        g.redis_client._client.zrem.assert_called_once_with(g.m_tracker_key, '42')
+        self.assertIsNone(reopened.get(42))
+
+    def test_redis_failure_keeps_success_on_disk(self):
+        outbox = BookingOutbox(str(Path(self.tmp.name) / 'redis-failure.sqlite3'))
+        outbox.put(42, 'tracker', {'status': 'grabbed'})
+        redis = MagicMock()
+        redis.set.side_effect = RuntimeError('redis offline')
+        outbox.flush(self.cloud, redis, lambda _: None)
+        self.assertIsNotNone(outbox.get(42))
+        redis.zrem.assert_not_called()
+        self.cloud.update_vas_task.assert_not_called()
+
+    def test_two_flushers_claim_each_success_only_once(self):
+        outbox = BookingOutbox(str(Path(self.tmp.name) / 'concurrent.sqlite3'))
+        outbox.put(42, 'tracker', {'status': 'grabbed'})
+        entered = threading.Event()
+        release = threading.Event()
+        self.cloud.update_vas_task.side_effect = lambda *_: (entered.set(), release.wait(3))
+        first = threading.Thread(target=outbox.flush, args=(self.cloud, MagicMock(), lambda _: None))
+        first.start()
+        try:
+            self.assertTrue(entered.wait(2))
+            reopened = BookingOutbox(outbox.path)
+            reopened.flush(self.cloud, MagicMock(), lambda _: None)
+            self.cloud.update_vas_task.assert_called_once()
+        finally:
+            release.set()
+            first.join(3)
+        self.assertFalse(first.is_alive())
+        self.assertIsNone(outbox.get(42))
+
+    def test_failed_terminal_status_update_retains_recovery_tracker(self):
+        g = self.make(OrderBookerGCO)
+        self.cloud.update_vas_task.side_effect = RuntimeError('cloud offline')
+        g._dispatch_error_policy('spawn', AccountIncompleteError(),
+                                 plg_cfg=PluginConfig(), task_id=42)
+        g.redis_client.zrem.assert_not_called()
+
+    def test_cleanup_retry_does_not_repeat_confirmed_cloud_update(self):
+        outbox = BookingOutbox(str(Path(self.tmp.name) / 'cleanup.sqlite3'))
+        outbox.put(42, 'tracker', {'status': 'grabbed'})
+        redis = MagicMock()
+        redis.zrem.side_effect = RuntimeError('redis offline')
+        outbox.flush(self.cloud, redis, lambda _: None)
+        self.assertIsNotNone(outbox.get(42))
+        redis.zrem.side_effect = None
+        with patch('utils.booking_outbox.time.time', return_value=time.time() + 10):
+            outbox.flush(self.cloud, redis, lambda _: None)
+        self.cloud.update_vas_task.assert_called_once()
+        self.assertIsNone(outbox.get(42))
+
+    def test_storage_outage_retains_in_memory_success_and_lease(self):
+        g = self.make()
+        g._leased_orders.add('42')
+        g.redis_client.set.return_value = False
+        with patch.object(g._outbox, 'put', side_effect=OSError('disk unavailable')), \
+                patch('booker.notify_book_result'):
+            g._push_success_notification(42, 'order', BookResult(success=True))
+        self.assertIn('42', g._pending_results)
+        self.assertIn('42', g._leased_orders)
+        g.redis_client.set.return_value = True
+        g._sync_successes()
+        self.assertNotIn('42', g._pending_results)
+        self.assertNotIn('42', g._leased_orders)
+
+    def test_slow_route_does_not_block_other_route_or_duplicate_booking(self):
+        g = self.make()
+        g.m_cfg.appointment_types = [AppointmentType(routing_key=k) for k in ('a', 'b')]
+        g.m_tasks = [Task(instance=MagicMock(), acceptable_routing_keys=[k]) for k in ('a', 'b')]
+        g.redis_client.mget.return_value = [self.signal('a'), self.signal('b')]
+        entered = {k: threading.Event() for k in ('a', 'b')}
+        release = threading.Event()
+        calls = []
+
+        def job(task, result):
+            key = result.apt_type.routing_key
+            calls.append(key)
+            entered[key].set()
+            release.wait(3)
+
+        g._execute_book_job = job
+        try:
+            g._poll_booking_signals()
+            self.assertTrue(entered['a'].wait(2))
+            self.assertTrue(entered['b'].wait(2))
+            g._schedule_maintenance()
+            for task in g.m_tasks:
+                task.instance.keep_alive.assert_not_called()
+            g._poll_booking_signals()
+            self.assertCountEqual(calls, ['a', 'b'])
+            self.assertEqual(g.redis_client.mget.call_count, 1)
+        finally:
+            release.set()
+
+    def test_busy_browser_is_not_refreshed_by_maintenance(self):
+        g = self.make()
+        plugin = MagicMock()
+        task = Task(instance=plugin, busy=True)
+        g.m_tasks = [task]
+        g._schedule_maintenance()
+        plugin.keep_alive.assert_not_called()
+
+    def test_cleanup_waits_for_inflight_booking(self):
+        g = self.make()
+        plugin = MagicMock()
+        task = Task(instance=plugin, busy=True)
+        g.m_tasks = [task]
+        started = threading.Event()
+        release = threading.Event()
+        g._execute_book_job = lambda *_: (started.set(), release.wait(3))
+        g._executor.submit(g._run_book_job, task, QueryResult())
+        self.assertTrue(started.wait(2))
+        stopper = threading.Thread(target=g.stop)
+        stopper.start()
+        try:
+            self.assertTrue(g.m_stop_event.wait(2))
+            plugin.cleanup.assert_not_called()
+        finally:
+            release.set()
+            stopper.join(3)
+        self.assertFalse(stopper.is_alive())
+        plugin.cleanup.assert_called_once()
+
+    def test_spawn_finishing_after_stop_is_cleaned_in_all_pool_modes(self):
+        for cls, method in ((SentinelGCO, '_spawn_sentinel_worker'),
+                            (BuiltinBookerGCO, '_spawn_worker'),
+                            (OrderBookerGCO, '_spawn_worker')):
+            with self.subTest(cls=cls):
+                g = self.make(cls)
+                self.cloud.get_vas_task_pop.return_value = {'id': 42, 'user_inputs': {'username': 'audit'}}
+                plugin = MagicMock()
+                plugin.create_session.side_effect = g.stop
+                g.m_factory.create = MagicMock(return_value=plugin)
+                getattr(g, method)(*(['a'] if cls is OrderBookerGCO else []))
+                self.assertEqual(g.m_tasks, [])
+                # Forced shutdown and late creator completion may both clean;
+                # cleanup is idempotent and ownership must be fully released.
+                plugin.cleanup.assert_called()
+                self.assertEqual(g._resources._entries, {})
+
+    def test_empty_group_does_not_poll_redis(self):
+        g = self.make()
+        g.m_cfg.appointment_types = [AppointmentType(routing_key='a')]
+        g._poll_booking_signals()
+        g.redis_client.mget.assert_not_called()
+
+    def test_signal_reads_are_batched(self):
+        g = self.make()
+        keys = ['a', 'b', 'c']
+        g.m_cfg.appointment_types = [AppointmentType(routing_key=k) for k in keys]
+        g.m_tasks = [Task(instance=MagicMock(), acceptable_routing_keys=keys)]
+        g.redis_client.mget.return_value = [None] * 3
+        g._poll_booking_signals()
+        g.redis_client.mget.assert_called_once_with(['vs:signal:' + k for k in keys])
+        g.redis_client.get.assert_not_called()
+
+    def test_available_signal_without_date_does_not_crash_or_book(self):
+        g = self.make(OrderBookerGCO)
+        result = QueryResult(availability_status=AvailabilityStatus.Available)
+        self.assertFalse(g._is_date_of_interest(Task(instance=MagicMock()), result))
+
+    def test_sentinel_slow_health_check_does_not_stall_other_instances(self):
+        g = self.make(SentinelGCO)
+        g.m_cfg.appointment_types = [AppointmentType(routing_key='a')]
+        slow = MagicMock(config=PluginConfig())
+        fast = MagicMock(config=PluginConfig())
+        entered = threading.Event()
+        release = threading.Event()
+        queried = threading.Event()
+        slow.health_check.side_effect = lambda: (entered.set(), release.wait(3), True)[-1]
+        slow.query.return_value = QueryResult()
+        fast.query.side_effect = lambda *_: (queried.set(), QueryResult())[-1]
+        g.m_tasks = [Task(instance=slow), Task(instance=fast)]
+        try:
+            with patch('sentinel.time.time', return_value=1000):
+                g._schedule_monitor_jobs()
+                self.assertTrue(entered.wait(2))
+            with patch('sentinel.time.time', return_value=1001):
+                deadline = time.monotonic() + 1
+                while g.m_tasks[1].busy and time.monotonic() < deadline:
+                    time.sleep(0.001)
+                g._schedule_monitor_jobs()
+                self.assertTrue(queried.wait(2))
+        finally:
+            release.set()
+
+    def test_executor_rejects_overload_instead_of_queuing(self):
+        executor = TaskExecutor(1, 'audit')
+        release = threading.Event()
+        try:
+            self.assertTrue(executor.submit(release.wait, 2))
+            self.assertFalse(executor.submit(lambda: None))
+        finally:
+            release.set()
+            executor.shutdown()
+
+
+class SweeperTests(unittest.TestCase):
+    def setUp(self):
+        self.sweeper = OrphanTaskSweeper({}, logger=MagicMock())
+        self.sweeper.redis_client = MagicMock()
+        self.sweeper._outbox = MagicMock()
+        self.sweeper._outbox.get.return_value = None
+        self.sweeper.redis_client.get.return_value = None
+        self.sweeper.redis_client.zscore.return_value = 0
+        self.cloud = MagicMock()
+        self.cloud.get_vas_task.return_value = {'status': 'processing'}
+        patcher = patch('toolkit.cloud_api.CloudApi.Instance', return_value=self.cloud)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_scan_instead_of_keys(self):
+        r = self.sweeper.redis_client
+        r.scan_iter.return_value = [b'tracker']
+        r.zrangebyscore.return_value = [b'42']
+        with patch.object(self.sweeper, '_rescue_task') as rescue:
+            self.sweeper._sweep()
+        r.keys.assert_not_called()
+        r.scan_iter.assert_called_once_with(match='vs:worker:tasks_tracker:*', count=100)
+        rescue.assert_called_once_with('tracker', 42)
+
+    def test_pending_success_is_never_requeued(self):
+        self.sweeper._outbox.get.return_value = {'status': 'grabbed'}
+        self.sweeper._rescue_task('tracker', 42)
+        self.cloud.return_vas_task_to_queue.assert_not_called()
+
+    def test_remote_success_marker_is_synchronized(self):
+        self.sweeper.redis_client.get.return_value = '{"status": "grabbed"}'
+        self.sweeper._rescue_task('tracker', 42)
+        self.cloud.update_vas_task.assert_called_once_with('42', {'status': 'grabbed'})
+        self.cloud.return_vas_task_to_queue.assert_not_called()
+
+    def test_lease_renewed_during_cloud_read_is_not_requeued(self):
+        self.sweeper.redis_client.zscore.side_effect = [0, time.time() + 120]
+        self.sweeper._rescue_task('tracker', 42)
+        self.cloud.return_vas_task_to_queue.assert_not_called()
+
+    def test_terminal_failed_order_is_not_requeued(self):
+        self.cloud.get_vas_task.return_value = {'status': 'failed'}
+        self.sweeper._rescue_task('tracker', 42)
+        self.cloud.return_vas_task_to_queue.assert_not_called()
+        self.sweeper.redis_client.zrem.assert_called_once_with('tracker', 42)
+
+
+class RedisTests(unittest.TestCase):
+    def test_heartbeat_uses_one_zadd_for_entire_mapping(self):
+        client = SafeRedisClient({})
+        client._client = MagicMock()
+        mapping = {str(i): 123 for i in range(100)}
+        self.assertTrue(client.bulk_zadd('tracker', mapping))
+        client._client.zadd.assert_called_once_with('tracker', mapping)
+        client._client.pipeline.assert_not_called()
+
+
+if __name__ == '__main__':
+    unittest.main()

+ 6 - 3
toolkit/cloud_api.py

@@ -19,8 +19,8 @@ class CloudApi:
     def __new__(cls, *args, **kwargs):
         if cls._instance is None:
             cls._instance = super(CloudApi, cls).__new__(cls)
-            cls._instance.base_url = "https://api.text.skin"
-            cls._instance.api_token = "Bearer tok_e946329a60ff45ba807f3f41b0e8b7fc"
+            cls._instance.base_url = configure.CLOUD_API_BASE_URL
+            cls._instance.api_token = configure.CLOUD_API_TOKEN
             cls._instance.session = requests.Session()
         return cls._instance
 
@@ -29,8 +29,11 @@ class CloudApi:
         return CloudApi()
 
     def _get_headers(self, content_type: str = "application/json") -> Dict[str, str]:
+        if not self.api_token:
+            raise BizLogicError("CLOUD_API_TOKEN is not configured")
+        token = self.api_token if self.api_token.startswith("Bearer ") else f"Bearer {self.api_token}"
         return {
-            "Authorization": self.api_token,
+            "Authorization": token,
             "Content-Type": content_type,
             "Accept": "application/json, text/plain, */*"
         }

+ 2 - 1
toolkit/mihomo_tunnel.py

@@ -134,6 +134,7 @@ class MihomoTunnel:
                 self.process.wait(timeout=3)
             except subprocess.TimeoutExpired:
                 self.process.kill()
+                self.process.wait(timeout=3)
         self.process = None
 
         if self.log_file_obj and not self.log_file_obj.closed:
@@ -193,4 +194,4 @@ if __name__ == "__main__":
         print(f"\nError: {e}")
     finally:
         tunnel.stop()
-        print("Tunnel closed and temp files cleaned.")
+        print("Tunnel closed and temp files cleaned.")

+ 56 - 0
toolkit/task_executor.py

@@ -0,0 +1,56 @@
+"""Bounded daemon workers with a finite shutdown wait.
+
+Python cannot safely terminate a blocked thread. Owners cancel their browser
+resources on timeout and retain a failed group until those calls unwind.
+"""
+import logging
+import queue
+import threading
+
+from utils.resource_lifecycle import join_threads
+
+
+class TaskExecutor:
+    def __init__(self, max_workers, name):
+        self._slots = threading.BoundedSemaphore(max_workers)
+        self._queue = queue.Queue(maxsize=max_workers)
+        self._lock = threading.Lock()
+        self._closed = threading.Event()
+        self._threads = []
+        self._max_workers = max_workers
+        self._name = name
+
+    def _worker(self):
+        while not self._closed.is_set() or not self._queue.empty():
+            try:
+                fn, args = self._queue.get(timeout=0.05)
+            except queue.Empty:
+                continue
+            try:
+                fn(*args)
+            except Exception:
+                logging.getLogger(__name__).exception('Background browser job failed')
+            finally:
+                self._slots.release()
+                self._queue.task_done()
+
+    def submit(self, fn, *args):
+        with self._lock:
+            if self._closed.is_set() or not self._slots.acquire(blocking=False):
+                return False
+            try:
+                if len(self._threads) < self._max_workers:
+                    thread = threading.Thread(target=self._worker, daemon=True,
+                                              name=f'{self._name}-{len(self._threads)}')
+                    thread.start()
+                    self._threads.append(thread)
+                self._queue.put_nowait((fn, args))
+            except Exception:
+                self._slots.release()
+                raise
+            return True
+
+    def shutdown(self, timeout=5):
+        with self._lock:
+            self._closed.set()
+        return join_threads(self._threads, timeout)

+ 1 - 1
tools/plugin_tester.py

@@ -567,7 +567,7 @@ def resolve_config_path(config_arg: Optional[str]) -> Optional[str]:
     1. 传入路径存在则直接使用
     2. 检查相对于 project_root 的路径
     3. 检查 project_root/config/ 下同名文件
-    4. 回退到 project_root/config/test_conf.json 或 test_conf.json
+    4. 回退到 project_root/config/test_conf.yaml 或 test_conf.yaml
     """
     if config_arg:
         candidates = [

+ 75 - 0
utils/booking_outbox.py

@@ -0,0 +1,75 @@
+"""Durable booking results, retried by workers and the sweeper after restart.
+
+The directory must be persistent. Docker services share it through /app/data.
+A Redis marker also protects results when the sweeper runs on another host.
+"""
+import json
+import os
+import sqlite3
+import time
+from contextlib import closing
+from pathlib import Path
+
+
+class BookingOutbox:
+    def __init__(self, path=None):
+        self.path = path or os.getenv('BOOKING_OUTBOX_PATH') or str(
+            Path(__file__).resolve().parent.parent / 'data' / 'booking_outbox.sqlite3')
+
+    def _connect(self):
+        Path(self.path).parent.mkdir(parents=True, exist_ok=True)
+        db = sqlite3.connect(self.path, timeout=10)
+        db.execute('''CREATE TABLE IF NOT EXISTS booking_results (
+            task_id TEXT PRIMARY KEY, tracker TEXT NOT NULL, payload TEXT NOT NULL,
+            lease_until REAL NOT NULL DEFAULT 0, cloud_synced INTEGER NOT NULL DEFAULT 0)''')
+        return db
+
+    @staticmethod
+    def marker(task_id):
+        return f'vs:booking:success:{task_id}'
+
+    def put(self, task_id, tracker, payload):
+        with closing(self._connect()) as db, db:
+            db.execute('INSERT INTO booking_results(task_id, tracker, payload) VALUES (?, ?, ?) '
+                       'ON CONFLICT(task_id) DO NOTHING',
+                       (str(task_id), tracker, json.dumps(payload)))
+
+    def get(self, task_id):
+        with closing(self._connect()) as db:
+            row = db.execute('SELECT payload FROM booking_results WHERE task_id=?',
+                             (str(task_id),)).fetchone()
+        return json.loads(row[0]) if row else None
+
+    def flush(self, cloud, redis_client, logger, tracker=None):
+        with closing(self._connect()) as db:
+            rows = db.execute('SELECT task_id, tracker, payload, cloud_synced FROM booking_results '
+                              'WHERE lease_until < ?' + (' AND tracker=?' if tracker else '') +
+                              ' ORDER BY lease_until LIMIT 20',
+                              (time.time(), tracker) if tracker else (time.time(),)).fetchall()
+        for task_id, tracker_key, raw, cloud_synced in rows:
+            # Separate processes sharing data must not send the same result concurrently.
+            with closing(self._connect()) as db, db:
+                claimed = db.execute('UPDATE booking_results SET lease_until=? '
+                                     'WHERE task_id=? AND lease_until < ?',
+                                     (time.time() + 60, task_id, time.time())).rowcount
+                if claimed:
+                    cloud_synced = db.execute('SELECT cloud_synced FROM booking_results WHERE task_id=?',
+                                              (task_id,)).fetchone()[0]
+            if not claimed:
+                continue
+            try:
+                redis_client.set(self.marker(task_id), raw)
+                if not cloud_synced:
+                    cloud.update_vas_task(task_id, json.loads(raw))
+                    with closing(self._connect()) as db, db:
+                        db.execute('UPDATE booking_results SET cloud_synced=1 WHERE task_id=?', (task_id,))
+                # Keep the journal if Redis is unavailable; cleanup is retryable too.
+                redis_client.zrem(tracker_key, task_id)
+                redis_client.delete(self.marker(task_id))
+                with closing(self._connect()) as db, db:
+                    db.execute('DELETE FROM booking_results WHERE task_id=?', (task_id,))
+            except Exception as exc:
+                logger(f'Booking result sync pending for task={task_id}: {exc}')
+                with closing(self._connect()) as db, db:
+                    db.execute('UPDATE booking_results SET lease_until=? WHERE task_id=?',
+                               (time.time() + 5, task_id))

+ 0 - 29
utils/browser_util.py

@@ -1,29 +0,0 @@
-from DrissionPage import ChromiumPage, ChromiumOptions
-
-
-def open_browser():
-    options = ChromiumOptions()
-    options.set_local_port(28888)
-    options.set_argument(arg='--disable-notifications', value=True)
-    options.set_argument(arg='--disable-extensions', value=False)
-    options.set_argument(arg='--disable-background-networking', value=True)
-    options.set_argument(arg='--disable-gpu', value=True)
-    options.set_argument(arg='--dns-prefetch-disable', value=True)
-    options.set_argument(arg='--disable-infobars', value=True)
-    options.set_argument(arg='--disable-dev-shm-usage', value=True)
-    options.set_argument(arg='--enable-automation', value=False)
-    options.set_argument(arg='--log-level', value='3')
-    options.set_argument(arg='--start-maximized', value=True)
-    options.set_argument(arg='--ignore-ssl-errors', value=True)
-    options.set_argument(arg='--no-sandbox')
-    options.headless(False)
-    options.incognito(False)
-    driver = ChromiumPage(addr_or_opts=options)
-    driver.set.window.max()
-    return driver
-
-def attach_browser():
-    return ChromiumPage(addr_or_opts=28888)
-
-def quit_browser(driver):
-    driver.quit()

+ 0 - 29
utils/browser_utils.py

@@ -1,29 +0,0 @@
-from DrissionPage import ChromiumPage, ChromiumOptions
-
-
-def open_browser():
-    options = ChromiumOptions()
-    options.set_local_port(28888)
-    options.set_argument(arg='--disable-notifications', value=True)
-    options.set_argument(arg='--disable-extensions', value=False)
-    options.set_argument(arg='--disable-background-networking', value=True)
-    options.set_argument(arg='--disable-gpu', value=True)
-    options.set_argument(arg='--dns-prefetch-disable', value=True)
-    options.set_argument(arg='--disable-infobars', value=True)
-    options.set_argument(arg='--disable-dev-shm-usage', value=True)
-    options.set_argument(arg='--enable-automation', value=False)
-    options.set_argument(arg='--log-level', value='3')
-    options.set_argument(arg='--start-maximized', value=True)
-    options.set_argument(arg='--ignore-ssl-errors', value=True)
-    options.set_argument(arg='--no-sandbox')
-    options.headless(False)
-    options.incognito(False)
-    driver = ChromiumPage(addr_or_opts=options)
-    driver.set.window.max()
-    return driver
-
-def attach_browser():
-    return ChromiumPage(addr_or_opts=28888)
-
-def quit_browser(driver):
-    driver.quit()

+ 59 - 20
utils/plugin_utils.py

@@ -460,31 +460,70 @@ def cleanup_browser_workspace(
         if logger:
             logger(msg)
 
-    # 1. 销毁浏览器
-    if page:
+    errors = []
+    if root_workspace is not None:
+        root_workspace = os.fspath(root_workspace)
+    browser_closed = page is None
+    process = None
+    if page is not None:
         try:
-            page.quit(force=True)
+            import psutil
+            pid = page.process_id
+            if isinstance(pid, int) and not isinstance(pid, bool) and pid > 0 and pid != os.getpid():
+                process = psutil.Process(pid)
+        except psutil.NoSuchProcess:
+            browser_closed = True
         except Exception:
             pass
-
-    # 2. 清理临时用户数据目录 (带多次重试与容错)
-    if root_workspace and os.path.exists(root_workspace):
-        for _ in range(3):
+        for attempt in range(2):
             try:
-                time.sleep(0.3)
-                shutil.rmtree(root_workspace, ignore_errors=True)
-                if not os.path.exists(root_workspace):
+                page.quit(force=True, timeout=1)
+                browser_closed = True
+                break
+            except Exception as exc:
+                if process is not None and not process.is_running():
+                    browser_closed = True
                     break
-            except BaseException:
-                pass
+                _log(f"Browser quit attempt {attempt + 1} failed: {exc}")
+        if process is not None and process.is_running():
             try:
-                time.sleep(0.3)
-            except BaseException:
-                pass
-
-    # 3. 停止代理隧道
-    if tunnel:
+                # Only the PID supplied by this page and its children are owned.
+                # psutil.Process also detects PID reuse through creation time.
+                children = process.children(recursive=True)
+                owned = children + [process]
+                for child in owned:
+                    try:
+                        child.terminate()
+                    except psutil.NoSuchProcess:
+                        pass
+                _, alive = psutil.wait_procs(owned, timeout=0.5)
+                for child in alive:
+                    child.kill()
+                _, alive = psutil.wait_procs(alive, timeout=0.5)
+                browser_closed = not alive
+            except Exception as exc:
+                browser_closed = False
+                errors.append(f"Owned browser process shutdown failed: {exc}")
+        if not browser_closed:
+            errors.append("Browser did not confirm shutdown; retaining its handle")
+
+    if tunnel is not None:
         try:
             tunnel.stop()
-        except Exception:
-            pass
+        except Exception as exc:
+            errors.append(f"Tunnel shutdown failed: {exc}")
+
+    if browser_closed and root_workspace and os.path.exists(root_workspace):
+        for _ in range(3):
+            try:
+                shutil.rmtree(root_workspace)
+                break
+            except FileNotFoundError:
+                break
+            except OSError:
+                time.sleep(0.1)
+        if os.path.exists(root_workspace):
+            errors.append(f"Browser workspace still exists: {root_workspace}")
+
+    if errors:
+        raise RuntimeError("; ".join(errors))

+ 16 - 0
utils/redis_stream_logger.py

@@ -63,6 +63,8 @@ class RedisStreamLogger:
         return cls._instance
 
     def _get_client(self) -> redis.Redis:
+        if self._stopped:
+            raise RuntimeError('Stream logger is closed')
         if self._redis_client is None:
             self._redis_client = redis.Redis(**self.redis_conf)
         return self._redis_client
@@ -140,10 +142,24 @@ class RedisStreamLogger:
             pipe.execute()
         except Exception:
             # Release client connection on error to allow reconnection
+            if self._redis_client is not None:
+                self._redis_client.close()
             self._redis_client = None
 
     def close(self):
         self._stopped = True
+        if self._worker_thread is not threading.current_thread():
+            self._worker_thread.join(2)
+        client = self._redis_client
+        self._redis_client = None
+        if client is not None:
+            client.close()
+            client.connection_pool.disconnect()
+        if self._worker_thread is not threading.current_thread():
+            self._worker_thread.join(1)
+        with type(self)._lock:
+            if type(self)._instance is self:
+                type(self)._instance = None
 
 
 class RedisStreamLogHandler(logging.Handler):

+ 168 - 0
utils/resource_lifecycle.py

@@ -0,0 +1,168 @@
+"""Keep ownership until cleanup succeeds, including partially initialized plugins."""
+import threading
+import time
+from functools import wraps
+from contextlib import contextmanager
+
+
+def init_lifecycle(owner):
+    owner._lifecycle_lock = threading.RLock()
+    owner._started = False
+    owner._fully_stopped = False
+    owner._resources = InstanceRegistry(owner._log)
+
+
+def start_once(method):
+    @wraps(method)
+    def start(owner):
+        with owner._lifecycle_lock:
+            if owner.m_stop_event.is_set():
+                raise RuntimeError('A stopped group cannot be restarted; create a new group')
+            if owner._started:
+                return
+            owner._started = True
+            try:
+                return method(owner)
+            except Exception:
+                try:
+                    owner.stop()
+                except Exception as exc:
+                    owner._log(f'Startup rollback pending: {exc}')
+                raise
+    return start
+
+
+def serialized_stop(method):
+    @wraps(method)
+    def stop(owner):
+        with owner._lifecycle_lock:
+            if owner._fully_stopped:
+                return
+            method(owner)
+            owner._fully_stopped = True
+    return stop
+
+
+class InstanceRegistry:
+    def __init__(self, logger):
+        self._logger = logger
+        self._lock = threading.RLock()
+        self._entries = {}
+        self._closing = False
+
+    @property
+    def empty(self):
+        with self._lock:
+            return not self._entries
+
+    def track(self, instance):
+        with self._lock:
+            return self._entries.setdefault(id(instance), {
+                'instance': instance, 'active': 0, 'thread': None,
+                'cleaned': False, 'failed': False})
+
+    @contextmanager
+    def operation(self, instance):
+        entry = self.track(instance)
+        with self._lock:
+            entry['active'] += 1
+            closing = self._closing
+        try:
+            if closing:
+                raise RuntimeError('Group is stopping')
+            yield
+        finally:
+            with self._lock:
+                entry['active'] -= 1
+                closing = self._closing
+                if entry['cleaned'] and not entry['active']:
+                    self._entries.pop(id(instance), None)
+            if closing:
+                self.cleanup(instance, force=True)
+
+    def cleanup(self, instance, force=False, timeout=3):
+        entry = self.track(instance)
+        with self._lock:
+            thread = entry['thread']
+            force_thread = entry.get('force_thread')
+            if (force and thread and thread.is_alive()
+                    and (not force_thread or not force_thread.is_alive())):
+                def force_release():
+                    try:
+                        from utils.plugin_utils import cleanup_browser_workspace
+                        cleanup_browser_workspace(**entry['handles'], logger=self._logger)
+                    except Exception as exc:
+                        self._logger(f'Forced browser shutdown pending: {exc}')
+                force_thread = threading.Thread(target=force_release, daemon=True,
+                                                name='Instance-Force-Cleanup')
+                entry['force_thread'] = force_thread
+                force_thread.start()
+            if not thread or not thread.is_alive():
+                entry['cleaned'] = False
+                # Keep handles even if a plugin clears its attributes on failure.
+                previous = entry.get('handles', {})
+                entry['handles'] = {name: getattr(instance, name, None) or previous.get(name)
+                                    for name in ('page', 'tunnel', 'root_workspace')}
+
+                def run():
+                    try:
+                        if force:
+                            instance.keep_browser_on_error = False
+                            from utils.plugin_utils import cleanup_browser_workspace
+                            cleanup_browser_workspace(**entry['handles'], logger=self._logger)
+                            for name in ('page', 'tunnel'):
+                                if getattr(instance, name, None) is entry['handles'][name]:
+                                    setattr(instance, name, None)
+                        elif getattr(instance, 'keep_browser_on_error', False) is True:
+                            raise RuntimeError('Diagnostic browser retained until explicit stop')
+                        instance.cleanup()
+                        with self._lock:
+                            entry['cleaned'] = True
+                            entry['failed'] = False
+                            if not entry['active']:
+                                self._entries.pop(id(instance), None)
+                    except Exception as exc:
+                        with self._lock:
+                            entry['failed'] = True
+                        self._logger(f'Instance cleanup pending: {exc}')
+
+                thread = threading.Thread(target=run, daemon=True, name='Instance-Cleanup')
+                entry['thread'] = thread
+                thread.start()
+        if thread is not threading.current_thread():
+            thread.join(max(0, timeout))
+        with self._lock:
+            if not entry['cleaned']:
+                entry['failed'] = True
+            return entry['cleaned']
+
+    def retry_failed(self, timeout=0):
+        with self._lock:
+            failed = [e['instance'] for e in self._entries.values() if e['failed']]
+        for instance in failed:
+            self.cleanup(instance, timeout=timeout)
+        with self._lock:
+            return not any(e['failed'] for e in self._entries.values())
+
+    def close(self, timeout=3):
+        with self._lock:
+            self._closing = True
+            entries = list(self._entries.values())
+        deadline = time.monotonic() + timeout
+        for entry in entries:
+            self.cleanup(entry['instance'], force=True, timeout=0)
+        for entry in entries:
+            if entry['thread']:
+                entry['thread'].join(max(0, deadline - time.monotonic()))
+            if entry.get('force_thread'):
+                entry['force_thread'].join(max(0, deadline - time.monotonic()))
+        with self._lock:
+            return not self._entries
+
+
+def join_threads(threads, timeout):
+    deadline = time.monotonic() + max(0, timeout)
+    for thread in threads:
+        if thread is not threading.current_thread() and thread.ident is not None:
+            thread.join(max(0, deadline - time.monotonic()))
+    return not any(t.is_alive() and t is not threading.current_thread() for t in threads)

+ 17 - 5
utils/safe_redis_cli.py

@@ -13,6 +13,12 @@ class SafeRedisClient:
         
         self._client = redis.Redis(**safe_conf)
         self._logger = logger
+        self._closed = False
+
+    def close(self):
+        self._closed = True
+        self._client.close()
+        self._client.connection_pool.disconnect()
 
     def _log(self, msg: str):
         if self._logger:
@@ -22,12 +28,16 @@ class SafeRedisClient:
 
     def _safe_execute(self, func, default_return, *args, **kwargs):
         """General safe executor: auto-retries once on error, returns safe fallback on total failure."""
+        if self._closed:
+            return default_return
         try:
             return func(*args, **kwargs)
         except Exception as e:
             func_name = getattr(func, '__name__', 'operation')
             self._log(f"Attempt 1 failed ({func_name}): {e}. Retrying in 0.5s...")
             time.sleep(0.5)
+            if self._closed:
+                return default_return
             try:
                 return func(*args, **kwargs)
             except Exception as e2:
@@ -40,6 +50,11 @@ class SafeRedisClient:
     def get(self, name) -> Any:
         return self._safe_execute(self._client.get, None, name)
 
+    def mget(self, names):
+        if not names:
+            return []
+        return self._safe_execute(self._client.mget, [None] * len(names), names)
+
     def set(self, name, value, ex=None) -> bool:
         return self._safe_execute(self._client.set, False, name, value, ex=ex)
 
@@ -67,9 +82,6 @@ class SafeRedisClient:
         if not mapping:
             return True
         def _op():
-            pipe = self._client.pipeline()
-            for k, v in mapping.items():
-                pipe.zadd(name, {k: v})
-            pipe.execute()
+            self._client.zadd(name, mapping)
             return True
-        return self._safe_execute(_op, False)
+        return self._safe_execute(_op, False)