|
@@ -8,7 +8,9 @@ from typing import List, Dict, Callable, Optional
|
|
|
|
|
|
|
|
from vs_types import (
|
|
from vs_types import (
|
|
|
GroupConfig, VSPlgConfig, Task, VSBookResult, VSQueryResult,
|
|
GroupConfig, VSPlgConfig, Task, VSBookResult, VSQueryResult,
|
|
|
- AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str
|
|
|
|
|
|
|
+ AppointmentType, QueryWaitMode, AvailabilityStatus, GroupStatus, PluginStatus, utc_now_str,
|
|
|
|
|
+ BizException, RateLimiteddError, SessionExpiredOrInvalidError, PermissionDeniedError,
|
|
|
|
|
+ NetworkTimeoutError
|
|
|
)
|
|
)
|
|
|
from vs_plg_factory import VSPlgFactory
|
|
from vs_plg_factory import VSPlgFactory
|
|
|
from toolkit.vs_cloud_api import VSCloudApi
|
|
from toolkit.vs_cloud_api import VSCloudApi
|
|
@@ -148,6 +150,109 @@ class BaseBookerGCO:
|
|
|
def _get_redis_key(self, routing_key: str) -> str:
|
|
def _get_redis_key(self, routing_key: str) -> str:
|
|
|
return f"vs:signal:{routing_key}"
|
|
return f"vs:signal:{routing_key}"
|
|
|
|
|
|
|
|
|
|
+ def _normalize_error(self, err: Exception) -> BizException:
|
|
|
|
|
+ """规整化异常为领域异常 BizException,保留对历史特征字符串的兜底兼容"""
|
|
|
|
|
+ if isinstance(err, BizException):
|
|
|
|
|
+ return err
|
|
|
|
|
+ err_str = str(err)
|
|
|
|
|
+ if "42901" in err_str or "Rate limited" in err_str:
|
|
|
|
|
+ return RateLimiteddError(message=err_str)
|
|
|
|
|
+ if "40101" in err_str or "Session expired" in err_str or "session invalid" in err_str.lower():
|
|
|
|
|
+ return SessionExpiredOrInvalidError(message=err_str)
|
|
|
|
|
+ if "40301" in err_str or "403" in err_str or "Cloudflare Access Denied" in err_str:
|
|
|
|
|
+ return PermissionDeniedError(message=err_str)
|
|
|
|
|
+ return BizException(
|
|
|
|
|
+ code=50000,
|
|
|
|
|
+ message=f"Unhandled ({type(err).__name__}): {err_str}",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ def _dispatch_error_policy(
|
|
|
|
|
+ self,
|
|
|
|
|
+ action: str,
|
|
|
|
|
+ err: Exception,
|
|
|
|
|
+ task: Optional[Task] = None,
|
|
|
|
|
+ plg_cfg: Optional[VSPlgConfig] = None,
|
|
|
|
|
+ task_id: Optional[int] = None
|
|
|
|
|
+ ) -> None:
|
|
|
|
|
+ """
|
|
|
|
|
+ Booker 错误调度与自愈策略引擎 (Policy Engine):
|
|
|
|
|
+ 基于 (Action, 异常事实类型) 矩阵 + Booker 模式闭环执行自愈动作:
|
|
|
|
|
+ - spawn/login 时 429:判定为账号级限流,锁定账号,更新 Redis 订单/任务追踪器与生成冷却;
|
|
|
|
|
+ - book 时 429:档期提交频控或出口 IP 限制,不锁定账号,移除任务以轮换代理或重试;
|
|
|
|
|
+ - session expired / 401:会话失效,移除任务;
|
|
|
|
|
+ - 403 Forbidden / WAF:判定为 IP 受阻,移除任务以轮换代理;
|
|
|
|
|
+ - 内部直接完成日志输出与定时器维护,调用方无需感知具体冷却数值。
|
|
|
|
|
+ """
|
|
|
|
|
+ biz_err = self._normalize_error(err)
|
|
|
|
|
+ acc = None
|
|
|
|
|
+ if plg_cfg and plg_cfg.account:
|
|
|
|
|
+ acc = plg_cfg.account
|
|
|
|
|
+ elif task and getattr(task.instance, "config", None) and task.instance.config.account:
|
|
|
|
|
+ acc = task.instance.config.account
|
|
|
|
|
+
|
|
|
|
|
+ # 1. 目标站限流 / 429
|
|
|
|
|
+ if isinstance(biz_err, RateLimiteddError):
|
|
|
|
|
+ if action in ("spawn", "login", "create_session"):
|
|
|
|
|
+ backoff = float(self.m_cfg.login_backoff)
|
|
|
|
|
+ self._log(f"🔒 [Account Rate Limited] 429 throttled during {action} (Account credential/frequency restricted).")
|
|
|
|
|
+ if acc and getattr(acc, "username", "") not in ("", "Guest") and getattr(acc, "id", 0):
|
|
|
|
|
+ try:
|
|
|
|
|
+ VSCloudApi.Instance().lock_account(acc.id, int(backoff))
|
|
|
|
|
+ self._log(f"🔒 Account [{acc.username}] locked for {int(backoff)}s via Cloud API.")
|
|
|
|
|
+ except Exception as lock_err:
|
|
|
|
|
+ self._log(f"Failed to lock account via cloud API: {lock_err}")
|
|
|
|
|
+
|
|
|
|
|
+ effective_task_id = task_id if task_id is not None else (task.task_ref if task else None)
|
|
|
|
|
+ if effective_task_id is not None:
|
|
|
|
|
+ self.redis_client.zadd(self.m_tracker_key, {str(effective_task_id): time.time() + self.m_cfg.login_backoff})
|
|
|
|
|
+
|
|
|
|
|
+ self.m_spawn_cooldown_until = time.time() + backoff
|
|
|
|
|
+ if task:
|
|
|
|
|
+ self._remove_task(task, "spawn account rate limited")
|
|
|
|
|
+
|
|
|
|
|
+ elif action == "book":
|
|
|
|
|
+ # 预订提交时 429:接口频控或 IP 限制,绝不锁账号,移除实例以更换代理重试
|
|
|
|
|
+ self._log("🔄 [Booking Rate Limited] 429 during booking (Slot/IP throttled, Account preserved). Removing task to rotate proxy...")
|
|
|
|
|
+ if task:
|
|
|
|
|
+ self._remove_task(task, "booking rate limited")
|
|
|
|
|
+ else:
|
|
|
|
|
+ self._log(f"⚠️ [Rate Limited] 429 encountered during {action}.")
|
|
|
|
|
+ if task:
|
|
|
|
|
+ self._remove_task(task, f"{action} rate limited")
|
|
|
|
|
+
|
|
|
|
|
+ # 2. 会话过期 / 401
|
|
|
|
|
+ elif isinstance(biz_err, SessionExpiredOrInvalidError):
|
|
|
|
|
+ if action in ("spawn", "login", "create_session"):
|
|
|
|
|
+ self._log(f"❌ [Auth Failed] Booker spawn failed (401). Cooling down spawn for 5s...")
|
|
|
|
|
+ self.m_spawn_cooldown_until = time.time() + 5.0
|
|
|
|
|
+ else:
|
|
|
|
|
+ self._log(f"🔄 [Session Expired] Booker session died during {action}. Removing task...")
|
|
|
|
|
+ if task:
|
|
|
|
|
+ self._remove_task(task, f"{action} session expired")
|
|
|
|
|
+
|
|
|
|
|
+ # 3. 权限被拒 / WAF 拦截 / 403
|
|
|
|
|
+ elif isinstance(biz_err, PermissionDeniedError):
|
|
|
|
|
+ proxy_cd = float(getattr(self.m_cfg, "proxy_cd", 60.0) or 60.0)
|
|
|
|
|
+ self._log(f"🛡️ [WAF Blocked] 403 Forbidden during {action}. Removing task to rotate proxy...")
|
|
|
|
|
+ if task:
|
|
|
|
|
+ self._remove_task(task, f"{action} waf blocked")
|
|
|
|
|
+ if action == "spawn":
|
|
|
|
|
+ self.m_spawn_cooldown_until = time.time() + proxy_cd
|
|
|
|
|
+
|
|
|
|
|
+ # 4. 网络或浏览器超时
|
|
|
|
|
+ elif isinstance(biz_err, NetworkTimeoutError):
|
|
|
|
|
+ self._log(f"⏱️ [Timeout] Network/browser timeout during {action}.")
|
|
|
|
|
+ if task:
|
|
|
|
|
+ task.next_run = time.time() + 5.0
|
|
|
|
|
+
|
|
|
|
|
+ # 5. 兜底策略
|
|
|
|
|
+ else:
|
|
|
|
|
+ self._log(f"⚠️ [{action}] Unexpected error: {biz_err.message} (code={biz_err.code})")
|
|
|
|
|
+ if action == "spawn":
|
|
|
|
|
+ self.m_spawn_cooldown_until = time.time() + 10.0
|
|
|
|
|
+ elif task:
|
|
|
|
|
+ task.next_run = time.time() + 10.0
|
|
|
|
|
+
|
|
|
def _is_within_active_hours(self) -> bool:
|
|
def _is_within_active_hours(self) -> bool:
|
|
|
"""判断当前是否在允许创建实例的时间段内"""
|
|
"""判断当前是否在允许创建实例的时间段内"""
|
|
|
start_str = self.m_cfg.active_time_start
|
|
start_str = self.m_cfg.active_time_start
|
|
@@ -345,16 +450,7 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
success = True
|
|
success = True
|
|
|
self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
|
|
self._log(f"+++ Built-in Booker spawned: {plg_cfg.account.username}")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- err_str = str(e)
|
|
|
|
|
- self.m_spawn_cooldown_until = time.time() + 30.0
|
|
|
|
|
- self._log(f"Spawn failed: {err_str}. Backing off for 30s...")
|
|
|
|
|
- rate_limited_indicators = [
|
|
|
|
|
- "42901" in err_str,
|
|
|
|
|
- "Rate limited" in err_str
|
|
|
|
|
- ]
|
|
|
|
|
- if any(rate_limited_indicators):
|
|
|
|
|
- if plg_cfg and plg_cfg.account.username != "Guest":
|
|
|
|
|
- VSCloudApi.Instance().lock_account(plg_cfg.account.id, self.m_cfg.login_backoff)
|
|
|
|
|
|
|
+ self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg)
|
|
|
finally:
|
|
finally:
|
|
|
if not success and instance:
|
|
if not success and instance:
|
|
|
self._cleanup_instance(instance, "spawn failed")
|
|
self._cleanup_instance(instance, "spawn failed")
|
|
@@ -388,14 +484,7 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
else:
|
|
else:
|
|
|
self._log(f"❌ BOOK FAILED for Order: {order_id}")
|
|
self._log(f"❌ BOOK FAILED for Order: {order_id}")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- err_str = str(e)
|
|
|
|
|
- self._log(f"Exception during booking: {err_str}")
|
|
|
|
|
- rate_limited_indicators = [
|
|
|
|
|
- "42901" in err_str,
|
|
|
|
|
- "Rate limited" in err_str
|
|
|
|
|
- ]
|
|
|
|
|
- if any(rate_limited_indicators):
|
|
|
|
|
- self._remove_task(task, "booking rate limited")
|
|
|
|
|
|
|
+ self._dispatch_error_policy("book", e, task=task)
|
|
|
|
|
|
|
|
|
|
|
|
|
class OrderBookerGCO(BaseBookerGCO):
|
|
class OrderBookerGCO(BaseBookerGCO):
|
|
@@ -502,14 +591,7 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
|
|
self._log(f"❌ BOOK FAILED for Order: {order_id}. Will retry on next signal.")
|
|
|
|
|
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- err_str = str(e)
|
|
|
|
|
- self._log(f"Exception during booking: {err_str}")
|
|
|
|
|
- rate_limited_indicators = [
|
|
|
|
|
- "42901" in err_str,
|
|
|
|
|
- "Rate limited" in err_str
|
|
|
|
|
- ]
|
|
|
|
|
- if any(rate_limited_indicators):
|
|
|
|
|
- self._remove_task(task, "booking rate limited")
|
|
|
|
|
|
|
+ self._dispatch_error_policy("book", e, task=task)
|
|
|
|
|
|
|
|
def _creator_loop(self):
|
|
def _creator_loop(self):
|
|
|
self._log("Creator loop started.")
|
|
self._log("Creator loop started.")
|
|
@@ -581,17 +663,7 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
success = True
|
|
success = True
|
|
|
self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
|
|
self._log(f"+++ Order Booker spawned: {plg_cfg.account.username} (Target: {acceptable_keys})")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- err_str = str(e)
|
|
|
|
|
- self.m_spawn_cooldown_until = time.time() + 30.0
|
|
|
|
|
- self._log(f"Order Booker spawn failed: {err_str}. Backing off for 30s...")
|
|
|
|
|
- rate_limited_indicators = [
|
|
|
|
|
- "42901" in err_str,
|
|
|
|
|
- "Rate limited" in err_str
|
|
|
|
|
- ]
|
|
|
|
|
- if any(rate_limited_indicators):
|
|
|
|
|
- if task_id is not None:
|
|
|
|
|
- self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + self.m_cfg.login_backoff})
|
|
|
|
|
-
|
|
|
|
|
|
|
+ self._dispatch_error_policy("spawn", e, plg_cfg=plg_cfg, task_id=task_id)
|
|
|
finally:
|
|
finally:
|
|
|
if not success and instance:
|
|
if not success and instance:
|
|
|
self._cleanup_instance(instance, "spawn failed")
|
|
self._cleanup_instance(instance, "spawn failed")
|