#!/usr/bin/env python3 """Run a duration-bounded local Italy Query soak with no booking actions.""" from __future__ import annotations import argparse import json import os import random import stat import sys import time import uuid from datetime import datetime, timezone from pathlib import Path from typing import Callable, Mapping from DrissionPage.errors import BrowserConnectError, PageDisconnectedError REPO_ROOT = Path(__file__).resolve().parents[1] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) from plugins.ita_plugin import ( CALENDAR_QUERY_MODE, ItaAuthenticationChallengeError, ItaCalendarPreparationRequiredError, ItaCalendarTransientError, ItaPlugin, ItaTargetServiceLinkMissingError, ) from tools.debug_ita_query import ( _get_query_credentials, _is_nonempty_query_result, _is_weak_entry_signal, _popup_text_for_log, _prepare_query_profile, _print_booking_form_structure, _print_failure_context, build_query_config, resolve_debug_route, ) from tools.debug_ita_session import ( DebugConfigurationError, resolve_chrome_path, ) from plugins.ita_plugin import CalendarGateError, guard_calendar from toolkit.backoff import ExponentialBackoff from vs_types import BizLogicError, PermissionDeniedError, RateLimiteddError, SessionExpiredOrInvalidError DAY_SECONDS = 24 * 60 * 60 MIN_DURATION_SECONDS = 60 DEFAULT_MIN_WAIT_SECONDS = 60 DEFAULT_MAX_WAIT_SECONDS = 300 DEFAULT_MAX_SESSION_ATTEMPTS = 0 DEFAULT_SESSION_RESTART_WAIT_SECONDS = 60 MAX_CONSECUTIVE_TARGET_LINK_MISSES = 2 MAX_CONSECUTIVE_BROWSER_DISCONNECTS = 2 MAX_CONSECUTIVE_SESSION_EXPIRATIONS = 2 MAX_CONSECUTIVE_BROWSER_CONNECT_FAILURES = 2 SESSION_REAUTHENTICATION_WINDOW_SECONDS = 60 * 60 MAX_SESSION_REAUTHENTICATIONS_PER_WINDOW = 6 SOAK_STATUS_PATH = REPO_ROOT / "data" / "ita_soak" / "status.json" CALENDAR_MIN_WAIT_SECONDS = 300 _STATUS_KEYS = frozenset( { "state", "service_id", "routing_key", "duration_seconds", "elapsed_seconds", "probes", "none_available", "transient_errors", "consecutive_transient_errors", "session_attempts", "session_expirations", "consecutive_session_expirations", "session_reauthentications_last_hour", "browser_disconnects", "consecutive_browser_disconnects", "browser_connect_failures", "consecutive_browser_connect_failures", "started_at", "updated_at", "exit_code", "reason", "calendar", } ) class _SoakCancelled(Exception): """Internal control flow for a clean Ctrl+C.""" class ItaSoakStatusError(ValueError): """Raised when the private local soak status cannot be written safely.""" class ItaSoakBrowserConnectLimitError(RuntimeError): """Raised after two consecutive browser/CDP connection failures.""" class ItaSoakSessionAttemptLimitError(RuntimeError): """Raised when an optional explicit Session-attempt limit is reached.""" def calendar_soak_limits(env: Mapping[str, str]) -> dict[str, int]: """Parse the explicit Calendar experiment limits without reading secrets.""" defaults = { "duration_seconds": 900, "min_wait": CALENDAR_MIN_WAIT_SECONDS, "max_wait": CALENDAR_MIN_WAIT_SECONDS, "max_session_attempts": DEFAULT_MAX_SESSION_ATTEMPTS, "restart_wait": DEFAULT_SESSION_RESTART_WAIT_SECONDS, "restart_after_probes": 0, } limits = {} for key, default in defaults.items(): try: limits[key] = int(env.get(f"ITA_SOAK_{key.upper()}", str(default))) except (TypeError, ValueError): raise DebugConfigurationError(f"ITA_SOAK_{key.upper()} must be an integer") from None _validate_soak_limits(**{key: value for key, value in limits.items() if key != "restart_after_probes"}) if limits["min_wait"] < CALENDAR_MIN_WAIT_SECONDS: raise DebugConfigurationError("Calendar soak min_wait must be at least 300 seconds") if limits["restart_after_probes"] < 0: raise DebugConfigurationError("restart_after_probes must be nonnegative") return limits class CalendarSoakContext: """Calendar-specific evidence, composed with the existing soak lifecycle. No scheduler, OTP reader, account pool, or publisher lives here. The owner transfers a single already-prepared plugin and audit to run_soak(). """ def __init__(self, *, plugin, route, request_audit, before): self.initial_plugin = plugin self.route = route self.config = plugin.config self.before = before self.audit = request_audit self.audit.enable_time_slot_counting() preparation = self.audit.settled_snapshot() if (preparation["otpRequests"] != 1 or preparation["avantiPosts"] != 1 or preparation["finalRequests"] != 0): raise CalendarGateError("Calendar preparation exceeded the one-shot contract") self.preparation_counts = preparation plugin.free_config["query_mode"] = CALENDAR_QUERY_MODE self.config.free_config["query_mode"] = CALENDAR_QUERY_MODE self.checkpoint = plugin.capture_calendar_query_checkpoint(route.appointment_type()) self.counts = {key: 0 for key in preparation} self.last_counts = preparation self.query_before = None self.previous_scan = None self.positive_queries = 0 self.negative_queries = 0 self.restores = 0 self.audit_gaps = 0 self._gap_recorded = False self.reservation_verified = False self.after = None self.queries_in_current_session = 0 self.latest = None self.failure_reason = "" self.expiry_causes = {"configuredLifetime": 0, "authentication": 0} def _collect(self, *, allow_disconnected: bool = False): if self.audit is None: return None if self.audit.driver.is_running is not True: if not allow_disconnected: raise PageDisconnectedError("Calendar audit disconnected") if not self._gap_recorded: self.audit_gaps += 1 self._gap_recorded = True current = self.audit.retained_snapshot() else: current = self.audit.settled_snapshot() if set(current) != set(self.last_counts): raise CalendarGateError("Calendar soak request counter shape changed") delta = {key: current[key] - self.last_counts[key] for key in current} if any(type(value) is not int or value < 0 for value in delta.values()): raise CalendarGateError("Calendar soak request counters moved backwards") for key, value in delta.items(): self.counts[key] += value self.last_counts = current if any(self.counts[key] for key in ("otpRequests", "avantiPosts", "finalRequests")): raise CalendarGateError("Calendar soak observed a forbidden preparation/final request") return current def attach_restored_session(self, plugin): from tools.debug_ita_calendar_gate import CalendarRequestAudit self.audit = CalendarRequestAudit(plugin.page, include_time_slots=True) self.audit.start() self.last_counts = {key: 0 for key in self.counts} self._gap_recorded = False self.queries_in_current_session = 0 self.latest = None plugin.restore_calendar_query(self.route.appointment_type(), self.checkpoint) self._collect() self.restores += 1 print(f"ITA_SOAK_CALENDAR_RESTORED restores={self.restores} fresh_proof=true", flush=True) def begin_query(self, plugin): self.latest = None # A failed new probe must not expose an old date as current. self.failure_reason = "" self.previous_scan = plugin.last_calendar_query_scan self.query_before = self._collect() def record_query(self, plugin, result): from tools.debug_ita_calendar_gate import _query_request_delta, validate_production_query_result scan = plugin.last_calendar_query_scan if scan is self.previous_scan: raise CalendarGateError("Calendar soak reused old scan evidence") report = validate_production_query_result( result, scan, apt_type=self.route.appointment_type(), lookahead_months=self.checkpoint.route.calendar_scan.lookahead_months, ) after = self._collect() standard_keys = {"otpRequests", "avantiPosts", "availabilityRequests", "finalRequests"} report["queryRequestDelta"] = _query_request_delta( {key: self.query_before[key] for key in standard_keys}, {key: after[key] for key in standard_keys}, expected_availability=scan.scanned_months, ) report["automaticTimeSlots"] = after["timeSlotRequests"] - self.query_before["timeSlotRequests"] guard_calendar(plugin.page) if result.success: self.positive_queries += 1 else: self.negative_queries += 1 self.latest = report self.queries_in_current_session += 1 print("ITA_SOAK_CALENDAR_QUERY_RESULT=" + json.dumps(report, sort_keys=True), flush=True) def close_session(self): if self.audit is None: return audit = self.audit try: self._collect(allow_disconnected=True) finally: try: audit.stop() finally: self.audit = None def verify_end(self, plugin): from tools.debug_ita_calendar_gate import read_reservation_snapshot if plugin is None or not plugin.page: return self._collect(allow_disconnected=True) if self.audit is None or self.audit.driver.is_running is not True: return after = read_reservation_snapshot(plugin.page) self.after = after if after != self.before: raise CalendarGateError("Calendar soak reservation or waitlist counts changed") self.reservation_verified = True def safe_dict(self): return { "mode": "calendar", "positiveQueries": self.positive_queries, "negativeQueries": self.negative_queries, "restores": self.restores, "queryRequests": dict(self.counts), "auditGaps": self.audit_gaps, "preparationRequests": dict(self.preparation_counts), "expiryCauses": dict(self.expiry_causes), "reservationVerified": self.reservation_verified, "reservationBefore": {"reservations": self.before.reservations, "waitlist": self.before.waitlist}, "reservationAfter": ( {"reservations": self.after.reservations, "waitlist": self.after.waitlist} if self.after is not None else None ), "queriesInCurrentSession": self.queries_in_current_session, "latestQuery": self.latest, "failureReason": self.failure_reason, "automaticOtpTodo": "ITA-OTP-01", } def run_prepared_calendar_soak(env, *, plugin, route, request_audit, before) -> int: """Take ownership from Q-CAL after exactly one manual preparation.""" try: limits = calendar_soak_limits(env) context = CalendarSoakContext( plugin=plugin, route=route, request_audit=request_audit, before=before, ) except Exception as exc: print(f"ITA_SOAK_CALENDAR_SETUP_FAILED type={type(exc).__name__}", flush=True) cleanup_failed = False for action in (request_audit.stop, plugin.cleanup): try: action() except Exception: cleanup_failed = True return 3 if cleanup_failed else 1 return run_soak( env, apply=True, non_interactive=True, prepared_calendar=context, **limits, ) def _ensure_private_directory(path: Path) -> Path: if path.is_symlink(): raise ItaSoakStatusError("Italy soak status directory must not be a symlink") try: path.mkdir(mode=0o700, parents=True, exist_ok=True) except OSError as exc: raise ItaSoakStatusError( f"cannot create Italy soak status directory ({type(exc).__name__})" ) from exc if path.is_symlink() or not path.is_dir(): raise ItaSoakStatusError( "Italy soak status path must be a private directory" ) if os.name == "posix": try: path.chmod(0o700) mode = stat.S_IMODE(path.stat().st_mode) except OSError as exc: raise ItaSoakStatusError( f"cannot secure Italy soak status directory ({type(exc).__name__})" ) from exc if mode != 0o700: raise ItaSoakStatusError( "Italy soak status directory permissions must be 700" ) return path.resolve() def write_soak_status( body: Mapping[str, object], *, status_path: str | Path = SOAK_STATUS_PATH, ) -> Path: """Atomically write a metadata-only local status snapshot.""" unsupported = sorted(set(body) - _STATUS_KEYS) if unsupported: raise ItaSoakStatusError( "Italy soak status contains unsupported keys: " + ", ".join(unsupported) ) path = Path(status_path).expanduser() if not path.is_absolute(): path = REPO_ROOT / path directory = _ensure_private_directory(path.parent) resolved_path = directory / path.name encoded = json.dumps( dict(body), ensure_ascii=True, sort_keys=True, separators=(",", ":"), ).encode("utf-8") temporary = directory / ( f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" ) descriptor: int | None = None try: flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL flags |= getattr(os, "O_NOFOLLOW", 0) descriptor = os.open(temporary, flags, 0o600) if os.name == "posix": os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: descriptor = None handle.write(encoded) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, resolved_path) if os.name == "posix": resolved_path.chmod(0o600) if stat.S_IMODE(resolved_path.stat().st_mode) != 0o600: raise ItaSoakStatusError( "Italy soak status file permissions must be 600" ) except ItaSoakStatusError: raise except OSError as exc: raise ItaSoakStatusError( f"cannot write Italy soak status ({type(exc).__name__})" ) from exc finally: if descriptor is not None: os.close(descriptor) try: temporary.unlink() except FileNotFoundError: pass except OSError: pass return resolved_path def _validate_soak_limits( *, duration_seconds: int, min_wait: int, max_wait: int, max_session_attempts: int, restart_wait: int, ) -> None: if not MIN_DURATION_SECONDS <= duration_seconds <= DAY_SECONDS: raise DebugConfigurationError( f"duration_seconds must be between {MIN_DURATION_SECONDS} " f"and {DAY_SECONDS}" ) if min_wait < DEFAULT_MIN_WAIT_SECONDS: raise DebugConfigurationError( f"min_wait must be at least {DEFAULT_MIN_WAIT_SECONDS} seconds" ) if max_wait < min_wait: raise DebugConfigurationError("max_wait must be greater than min_wait") if max_session_attempts < 0: raise DebugConfigurationError( "max_session_attempts must be 0 (duration-bounded) or positive" ) if restart_wait < DEFAULT_SESSION_RESTART_WAIT_SECONDS: raise DebugConfigurationError( "restart_wait must be at least " f"{DEFAULT_SESSION_RESTART_WAIT_SECONDS} seconds" ) def _service_list_structure_for_log(plugin: ItaPlugin) -> str: raw = getattr(plugin, "last_service_list_structure", None) if not isinstance(raw, dict): return "not-available" allowed = { "path": raw.get("path"), "ready_state": raw.get("ready_state"), "table_count": int(raw.get("table_count") or 0), "row_count": int(raw.get("row_count") or 0), "booking_link_count": int(raw.get("booking_link_count") or 0), "target_link_count": int(raw.get("target_link_count") or 0), } return json.dumps(allowed, ensure_ascii=True, sort_keys=True) def run_soak( env: Mapping[str, str], *, apply: bool, non_interactive: bool, duration_seconds: int, min_wait: int, max_wait: int, max_session_attempts: int = DEFAULT_MAX_SESSION_ATTEMPTS, restart_wait: int = DEFAULT_SESSION_RESTART_WAIT_SECONDS, randint: Callable[[int, int], int] = random.randint, sleep: Callable[[float], None] = time.sleep, monotonic: Callable[[], float] = time.monotonic, wall_time: Callable[[], float] = time.time, plugin_factory: Callable[[str], ItaPlugin] | None = None, status_writer: Callable[[Mapping[str, object]], object] | None = None, prepared_calendar: CalendarSoakContext | None = None, restart_after_probes: int = 0, ) -> int: # The prepared instance owns its verified route. Do not reinterpret the # environment after ownership has transferred from the Calendar gate. route = prepared_calendar.route if prepared_calendar is not None else resolve_debug_route(env) route_log = ( f"service_id={route.service_id} routing_key={route.routing_key}" ) _validate_soak_limits( duration_seconds=duration_seconds, min_wait=min_wait, max_wait=max_wait, max_session_attempts=max_session_attempts, restart_wait=restart_wait, ) if prepared_calendar is not None and min_wait < CALENDAR_MIN_WAIT_SECONDS: raise DebugConfigurationError("Calendar soak min_wait must be at least 300 seconds") if type(restart_after_probes) is not int or restart_after_probes < 0: raise DebugConfigurationError("restart_after_probes must be a nonnegative integer") if restart_after_probes and prepared_calendar is None: raise DebugConfigurationError("Controlled restart requires the prepared Calendar mode") if not apply: print( "ITA_SOAK_PREFLIGHT_OK " f"{route_log} " f"duration_seconds={duration_seconds} " f"min_wait={min_wait} max_wait={max_wait} " f"max_session_attempts={max_session_attempts} " "external_request=false" ) return 0 if prepared_calendar is None: chrome_path = resolve_chrome_path(env) os.environ["CHROME_BIN"] = str(chrome_path) username, password = _get_query_credentials( env, non_interactive=non_interactive, ) profile_path = _prepare_query_profile(username) config = build_query_config( env, username=username, password=password, persistent_profile_path=profile_path, route=route, ) else: config = prepared_calendar.config factory = plugin_factory or ItaPlugin writer = status_writer or write_soak_status apt_type = route.appointment_type() monitor_started = monotonic() started_wall = wall_time() plugin: ItaPlugin | None = prepared_calendar.initial_plugin if prepared_calendar else None current_session = 1 if prepared_calendar else 0 probes = 0 none_available = 0 transient_errors = 0 consecutive_transient_errors = 0 session_attempts = current_session session_expirations = 0 consecutive_session_expirations = 0 session_reauthentication_times: list[float] = [] browser_disconnects = 0 consecutive_browser_disconnects = 0 browser_connect_failures = 0 consecutive_browser_connect_failures = 0 exit_code = 0 final_state = "running" final_reason = "" cleanup_error: Exception | None = None controlled_restart_done = False consecutive_calendar_network_errors = 0 network_backoff = ExponentialBackoff( base_delay=restart_wait, max_delay=max_wait, jitter_range=(0, 0), ) def elapsed_seconds() -> int: elapsed = max(0.0, monotonic() - monitor_started) return min(duration_seconds, int(elapsed)) def status_body(state: str, *, code: int | None = None) -> dict: now = wall_time() current_monotonic = monotonic() reauthentications_last_hour = sum( 1 for timestamp in session_reauthentication_times if 0 <= current_monotonic - timestamp < SESSION_REAUTHENTICATION_WINDOW_SECONDS ) body = { "state": state, "service_id": route.service_id, "routing_key": apt_type.routing_key, "duration_seconds": duration_seconds, "elapsed_seconds": elapsed_seconds(), "probes": probes, "none_available": none_available, "transient_errors": transient_errors, "consecutive_transient_errors": consecutive_transient_errors, "session_attempts": session_attempts, "session_expirations": session_expirations, "consecutive_session_expirations": ( consecutive_session_expirations ), "session_reauthentications_last_hour": ( reauthentications_last_hour ), "browser_disconnects": browser_disconnects, "consecutive_browser_disconnects": ( consecutive_browser_disconnects ), "browser_connect_failures": browser_connect_failures, "consecutive_browser_connect_failures": ( consecutive_browser_connect_failures ), "started_at": datetime.fromtimestamp( started_wall, tz=timezone.utc, ).isoformat(), "updated_at": datetime.fromtimestamp( now, tz=timezone.utc, ).isoformat(), } if code is not None: body["exit_code"] = code if final_reason: body["reason"] = final_reason if prepared_calendar is not None: body["calendar"] = prepared_calendar.safe_dict() return body def emit_status(state: str, *, code: int | None = None) -> None: writer(status_body(state, code=code)) def cleanup_session(reason: str) -> None: nonlocal plugin, cleanup_error if plugin is None: return instance = plugin plugin = None try: try: if prepared_calendar is not None: prepared_calendar.close_session() finally: instance.cleanup() except Exception as exc: cleanup_error = exc raise print( "ITA_SOAK_SESSION_CLEANUP_OK " f"session={current_session} reason={reason}", flush=True, ) def wait_before_session_restart() -> bool: remaining = duration_seconds - (monotonic() - monitor_started) wait_seconds = min(float(restart_wait), max(0.0, remaining)) if wait_seconds: print( "ITA_SOAK_SESSION_RESTART_WAIT " f"seconds={int(wait_seconds)}", flush=True, ) sleep(wait_seconds) return monotonic() - monitor_started < duration_seconds def start_session() -> bool: nonlocal plugin nonlocal current_session nonlocal session_attempts nonlocal transient_errors nonlocal browser_connect_failures nonlocal consecutive_browser_connect_failures while monotonic() - monitor_started < duration_seconds: if ( max_session_attempts > 0 and session_attempts >= max_session_attempts ): raise ItaSoakSessionAttemptLimitError session_attempts += 1 current_session = session_attempts instance = factory("ita-soak-local-debug") plugin = instance instance.set_log(lambda message: print(message, flush=True)) instance.set_config(config) try: instance.create_session() except BrowserConnectError: browser_connect_failures += 1 transient_errors += 1 consecutive_browser_connect_failures += 1 print( "ITA_SOAK_BROWSER_CONNECT_FAILED " f"session_attempt={current_session} " f"consecutive={consecutive_browser_connect_failures} " f"total={browser_connect_failures}", flush=True, ) cleanup_session("browser-connect-failed") if ( consecutive_browser_connect_failures >= MAX_CONSECUTIVE_BROWSER_CONNECT_FAILURES ): raise ItaSoakBrowserConnectLimitError if not wait_before_session_restart(): return False continue consecutive_browser_connect_failures = 0 if prepared_calendar is not None: prepared_calendar.attach_restored_session(instance) print( "ITA_SOAK_SESSION_STARTED " f"session={current_session} " f"session_attempts={session_attempts}", flush=True, ) return True return False try: if plugin is None: start_session() print( "ITA_SOAK_STARTED " f"{route_log} " f"duration_seconds={duration_seconds} " f"min_wait={min_wait} max_wait={max_wait}", flush=True, ) emit_status("running") while True: if monotonic() - monitor_started >= duration_seconds: final_state = "completed" print( "ITA_SOAK_COMPLETED " f"duration_seconds={duration_seconds} probes={probes} " f"none_available={none_available} " f"transient_errors={transient_errors} " f"session_attempts={session_attempts}", flush=True, ) break if (prepared_calendar is not None and restart_after_probes and not controlled_restart_done and prepared_calendar.positive_queries + prepared_calendar.negative_queries >= restart_after_probes): # A deliberate normal close tests ownership and restore, not # a fabricated website outage or an authentication expiry. controlled_restart_done = True emit_status("controlled_restart") cleanup_session("controlled-restart") if not wait_before_session_restart(): continue start_session() emit_status("running") continue probes += 1 print( "ITA_SOAK_PROBE_START " f"probe={probes} session={current_session} " f"elapsed_seconds={elapsed_seconds()}", flush=True, ) try: if prepared_calendar is not None: if not plugin.health_check(): if getattr(plugin, "_query_stop_reason", None) == "browser_disconnected": raise PageDisconnectedError("Calendar browser disconnected") raise SessionExpiredOrInvalidError("Calendar Session health expired") prepared_calendar.begin_query(plugin) result = plugin.query(apt_type) if prepared_calendar is not None: prepared_calendar.record_query(plugin, result) except ItaCalendarTransientError: if prepared_calendar is None: raise prepared_calendar.latest = None transient_errors += 1 consecutive_calendar_network_errors += 1 prepared_calendar.failure_reason = "transient-network" emit_status("cooldown") if consecutive_calendar_network_errors >= 2: final_state, final_reason, exit_code = "failed", "consecutive-calendar-network-errors", 1 break remaining = duration_seconds - (monotonic() - monitor_started) sleep(min(network_backoff.calculate(consecutive_calendar_network_errors), max(0.0, remaining))) continue except ItaTargetServiceLinkMissingError: consecutive_session_expirations = 0 consecutive_browser_disconnects = 0 transient_errors += 1 consecutive_transient_errors += 1 print( "ITA_SERVICE_LIST_STRUCTURE=" + _service_list_structure_for_log(plugin), flush=True, ) print( "ITA_SOAK_TRANSIENT " "type=ItaTargetServiceLinkMissingError " f"consecutive={consecutive_transient_errors} " f"total={transient_errors}", flush=True, ) if ( consecutive_transient_errors >= MAX_CONSECUTIVE_TARGET_LINK_MISSES ): final_state = "failed" final_reason = "consecutive-target-link-misses" exit_code = 1 print( "ITA_SOAK_FAILED " "reason=consecutive-target-link-misses", flush=True, ) break except PageDisconnectedError: if prepared_calendar is not None: prepared_calendar.latest = None prepared_calendar.failure_reason = "browser-disconnected" browser_disconnects += 1 transient_errors += 1 consecutive_browser_disconnects += 1 consecutive_transient_errors = 0 print( "ITA_SOAK_BROWSER_DISCONNECTED " f"session={current_session} " f"consecutive={consecutive_browser_disconnects} " f"total={browser_disconnects}", flush=True, ) cleanup_session("browser-disconnected") if ( consecutive_browser_disconnects >= MAX_CONSECUTIVE_BROWSER_DISCONNECTS ): final_state = "failed" final_reason = "consecutive-browser-disconnects" exit_code = 1 print( "ITA_SOAK_FAILED " "reason=consecutive-browser-disconnects", flush=True, ) break if not wait_before_session_restart(): continue start_session() emit_status("running") continue except SessionExpiredOrInvalidError: if prepared_calendar is not None: prepared_calendar.latest = None prepared_calendar.failure_reason = "session-expired" expiry_cause = ( "configuredLifetime" if getattr(plugin, "_query_stop_reason", None) == "configured_lifetime_reached" else "authentication" ) prepared_calendar.expiry_causes[expiry_cause] += 1 session_expirations += 1 consecutive_session_expirations += 1 consecutive_transient_errors = 0 consecutive_browser_disconnects = 0 current_monotonic = monotonic() session_reauthentication_times[:] = [ timestamp for timestamp in session_reauthentication_times if 0 <= current_monotonic - timestamp < SESSION_REAUTHENTICATION_WINDOW_SECONDS ] session_reauthentication_times.append(current_monotonic) reauthentications_last_hour = len( session_reauthentication_times ) print( "ITA_SOAK_SESSION_EXPIRED " f"session={current_session} " f"session_expirations={session_expirations} " f"consecutive={consecutive_session_expirations} " f"reauthentications_last_hour=" f"{reauthentications_last_hour}", flush=True, ) cleanup_session("session-expired") if ( consecutive_session_expirations >= MAX_CONSECUTIVE_SESSION_EXPIRATIONS ): final_state = "failed" final_reason = "consecutive-session-expirations" exit_code = 1 print( "ITA_SOAK_FAILED " "reason=consecutive-session-expirations", flush=True, ) break if ( reauthentications_last_hour > MAX_SESSION_REAUTHENTICATIONS_PER_WINDOW ): final_state = "failed" final_reason = "reauthentication-rate-limit" exit_code = 1 print( "ITA_SOAK_FAILED " "reason=reauthentication-rate-limit", flush=True, ) break if not wait_before_session_restart(): continue start_session() emit_status("running") continue else: consecutive_calendar_network_errors = 0 consecutive_transient_errors = 0 consecutive_session_expirations = 0 consecutive_browser_disconnects = 0 if prepared_calendar is not None: if not result.success: none_available += 1 elif _is_weak_entry_signal(result): _print_booking_form_structure(plugin, route=route) final_state = "weak_signal" final_reason = "booking-form" exit_code = 4 print( "\aITA_SOAK_WEAK_SIGNAL " f"{route_log} probe={probes} " "evidence=booking-form", flush=True, ) break elif _is_nonempty_query_result(result): final_state = "safe_stop" final_reason = "unexpected-non-empty-result" exit_code = 4 print( "ITA_SOAK_SAFE_STOP " "reason=unexpected-non-empty-result", flush=True, ) break if prepared_calendar is None: none_available += 1 print( "ITA_SOAK_NONE_AVAILABLE " f"probe={probes} session={current_session}", flush=True, ) emit_status("running") remaining = duration_seconds - (monotonic() - monitor_started) if remaining <= 0: continue wait_seconds = min(float(randint(min_wait, max_wait)), remaining) print( "ITA_SOAK_WAIT " f"next_probe={probes + 1} seconds={int(wait_seconds)}", flush=True, ) sleep(wait_seconds) except ItaSoakBrowserConnectLimitError: final_state = "failed" final_reason = "consecutive-browser-connect-failures" exit_code = 1 print( "ITA_SOAK_FAILED " "reason=consecutive-browser-connect-failures", flush=True, ) except ItaSoakSessionAttemptLimitError: final_state = "failed" final_reason = "max-session-attempts-reached" exit_code = 1 print( "ITA_SOAK_FAILED reason=max-session-attempts-reached", flush=True, ) except ItaCalendarPreparationRequiredError: final_state, final_reason, exit_code = "needs_manual", "calendar-preparation-required", 4 if prepared_calendar is not None: prepared_calendar.latest = None prepared_calendar.failure_reason = final_reason print("ITA_SOAK_NEEDS_MANUAL todo=ITA-OTP-01 otp_sent=false avanti_submitted=false", flush=True) except (PermissionDeniedError, ItaAuthenticationChallengeError): final_state, final_reason, exit_code = "needs_manual", "website-verification-required", 4 print("ITA_SOAK_NEEDS_MANUAL reason=website-verification-required", flush=True) except RateLimiteddError: final_state, final_reason, exit_code = "rate_limited", "server-rate-limit", 5 print("ITA_SOAK_RATE_LIMITED retry=false account_rotation=false", flush=True) except KeyboardInterrupt: final_state = "cancelled" final_reason = "operator-cancelled" exit_code = 130 print( "\nITA_SOAK_CANCELLED reason=operator-cancelled", flush=True, ) except (BizLogicError, CalendarGateError) as exc: final_state = "failed" final_reason = "business-error" exit_code = 1 print( f"ITA_SOAK_FAILED type={type(exc).__name__} " f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(exc)}", flush=True, ) if plugin is not None and prepared_calendar is None: _print_failure_context(plugin) except Exception as exc: final_state = "failed" final_reason = "unexpected-error" exit_code = 1 print( f"ITA_SOAK_FAILED type={type(exc).__name__} " f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(exc)}", flush=True, ) if plugin is not None and prepared_calendar is None: _print_failure_context(plugin) finally: if prepared_calendar is not None: if exit_code != 0: prepared_calendar.latest = None prepared_calendar.failure_reason = final_reason try: prepared_calendar.verify_end(plugin) except Exception as exc: print(f"ITA_SOAK_END_EVIDENCE_UNAVAILABLE type={type(exc).__name__}", flush=True) if exit_code == 0: has_queries = prepared_calendar.positive_queries + prepared_calendar.negative_queries > 0 if (not has_queries or prepared_calendar.audit_gaps or not prepared_calendar.reservation_verified or prepared_calendar.queries_in_current_session == 0 or consecutive_calendar_network_errors or consecutive_session_expirations or consecutive_browser_disconnects or (restart_after_probes and not controlled_restart_done)): final_state, final_reason, exit_code = "incomplete", "calendar-evidence-incomplete", 6 try: cleanup_session("soak-finished") except Exception as exc: cleanup_error = exc if cleanup_error is not None: final_state = "failed" final_reason = "cleanup-failed" print( f"ITA_SOAK_CLEANUP_FAILED type={type(cleanup_error).__name__} " f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(cleanup_error)}", file=sys.stderr, flush=True, ) else: print("ITA_SOAK_CLEANUP_OK", flush=True) final_code = 3 if cleanup_error is not None else exit_code if prepared_calendar is not None and final_code != 0: prepared_calendar.failure_reason = final_reason prepared_calendar.latest = None try: emit_status(final_state, code=final_code) except Exception as exc: cleanup_error = cleanup_error or exc if prepared_calendar is not None: final_state, final_reason = "failed", "status-write-failed" prepared_calendar.failure_reason = final_reason print( f"ITA_SOAK_STATUS_FAILED type={type(exc).__name__}", file=sys.stderr, flush=True, ) if prepared_calendar is not None: report = prepared_calendar.safe_dict() report.update({ "state": final_state, "reason": final_reason, "elapsedSeconds": max(0, int(monotonic() - monitor_started)), "queryAttempts": probes, "sessionAttempts": session_attempts, "controlledRestart": controlled_restart_done, "exitCode": 3 if cleanup_error is not None else exit_code, "passed": cleanup_error is None and exit_code == 0, }) print("ITA_SOAK_CALENDAR_RESULT=" + json.dumps(report, sort_keys=True), flush=True) return 3 if cleanup_error is not None else exit_code def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Duration-bounded route-configured Italy Query soak. Without --run, " "performs a zero-network preflight only." ) ) parser.add_argument( "--calendar", action="store_true", help="Use one manual Calendar preparation, then the production Query and read-only recovery.", ) parser.add_argument( "--run", action="store_true", help="Start the real website soak.", ) parser.add_argument( "--non-interactive", action="store_true", help="Fail instead of prompting when local credentials are unavailable.", ) parser.add_argument( "--duration-seconds", type=int, default=None, help=f"Soak duration in seconds (maximum: {DAY_SECONDS}).", ) parser.add_argument( "--min-wait", type=int, default=None, help=f"Minimum seconds between probes (Calendar floor: {CALENDAR_MIN_WAIT_SECONDS}; entry-probe floor: {DEFAULT_MIN_WAIT_SECONDS}).", ) parser.add_argument( "--max-wait", type=int, default=None, help="Maximum seconds between probes.", ) parser.add_argument( "--max-session-attempts", type=int, default=DEFAULT_MAX_SESSION_ATTEMPTS, help=( "Optional total Session-attempt limit; 0 uses the duration and " "reauthentication circuit breakers only." ), ) parser.add_argument( "--restart-wait", type=int, default=DEFAULT_SESSION_RESTART_WAIT_SECONDS, help=( "Minimum seconds before recreating an explicitly expired Session " f"(floor: {DEFAULT_SESSION_RESTART_WAIT_SECONDS})." ), ) parser.add_argument( "--restart-after-probes", type=int, default=0, help="Calendar only: normally close/recreate once after this many successful queries (0 disables).", ) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(argv) try: if args.calendar: from tools.debug_ita_calendar_gate import build_calendar_gate_config, run_calendar_gate env = dict(os.environ) for key in ( "duration_seconds", "min_wait", "max_wait", "max_session_attempts", "restart_wait", "restart_after_probes", ): value = getattr(args, key) if value is not None: env[f"ITA_SOAK_{key.upper()}"] = str(value) limits = calendar_soak_limits(env) # Configuration only: no credential lookup, profile creation, or login. build_calendar_gate_config( env, username="preflight@example.invalid", password="unused-preflight", ) if not args.run: print("ITA_SOAK_CALENDAR_PREFLIGHT_OK " + json.dumps(limits, sort_keys=True) + " external_request=false", flush=True) return 0 if args.non_interactive: raise DebugConfigurationError("Calendar preparation currently requires manual OTP input") env["ITA_QCAL_MODE"] = "calendar_soak" return run_calendar_gate(env) if args.restart_after_probes: raise DebugConfigurationError("--restart-after-probes requires --calendar") return run_soak( os.environ, apply=args.run, non_interactive=args.non_interactive, duration_seconds=args.duration_seconds if args.duration_seconds is not None else DAY_SECONDS, min_wait=args.min_wait if args.min_wait is not None else DEFAULT_MIN_WAIT_SECONDS, max_wait=args.max_wait if args.max_wait is not None else DEFAULT_MAX_WAIT_SECONDS, max_session_attempts=args.max_session_attempts, restart_wait=args.restart_wait, ) except (DebugConfigurationError, CalendarGateError) as exc: print(f"ITA_SOAK_CONFIG_ERROR: {exc}", file=sys.stderr) return 2 if __name__ == "__main__": raise SystemExit(main())