| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170 |
- #!/usr/bin/env python3
- """Run a duration-bounded local Italy Query soak with no booking actions."""
- from __future__ import annotations
- import argparse
- import json
- import os
- import random
- import stat
- import sys
- import time
- import uuid
- from datetime import datetime, timezone
- from pathlib import Path
- from typing import Callable, Mapping
- from DrissionPage.errors import BrowserConnectError, PageDisconnectedError
- REPO_ROOT = Path(__file__).resolve().parents[1]
- if str(REPO_ROOT) not in sys.path:
- sys.path.insert(0, str(REPO_ROOT))
- from plugins.ita_plugin import (
- CALENDAR_QUERY_MODE,
- ItaAuthenticationChallengeError,
- ItaCalendarPreparationRequiredError,
- ItaCalendarTransientError,
- ItaPlugin,
- ItaTargetServiceLinkMissingError,
- )
- from tools.debug_ita_query import (
- _get_query_credentials,
- _is_nonempty_query_result,
- _is_weak_entry_signal,
- _popup_text_for_log,
- _prepare_query_profile,
- _print_booking_form_structure,
- _print_failure_context,
- build_query_config,
- resolve_debug_route,
- )
- from tools.debug_ita_session import (
- DebugConfigurationError,
- resolve_chrome_path,
- )
- from plugins.ita_calendar_query import CalendarGateError, guard_calendar
- from toolkit.backoff import ExponentialBackoff
- from vs_types import BizLogicError, PermissionDeniedError, RateLimiteddError, SessionExpiredOrInvalidError
- DAY_SECONDS = 24 * 60 * 60
- MIN_DURATION_SECONDS = 60
- DEFAULT_MIN_WAIT_SECONDS = 60
- DEFAULT_MAX_WAIT_SECONDS = 300
- DEFAULT_MAX_SESSION_ATTEMPTS = 0
- DEFAULT_SESSION_RESTART_WAIT_SECONDS = 60
- MAX_CONSECUTIVE_TARGET_LINK_MISSES = 2
- MAX_CONSECUTIVE_BROWSER_DISCONNECTS = 2
- MAX_CONSECUTIVE_SESSION_EXPIRATIONS = 2
- MAX_CONSECUTIVE_BROWSER_CONNECT_FAILURES = 2
- SESSION_REAUTHENTICATION_WINDOW_SECONDS = 60 * 60
- MAX_SESSION_REAUTHENTICATIONS_PER_WINDOW = 6
- SOAK_STATUS_PATH = REPO_ROOT / "data" / "ita_soak" / "status.json"
- CALENDAR_MIN_WAIT_SECONDS = 300
- _STATUS_KEYS = frozenset(
- {
- "state",
- "service_id",
- "routing_key",
- "duration_seconds",
- "elapsed_seconds",
- "probes",
- "none_available",
- "transient_errors",
- "consecutive_transient_errors",
- "session_attempts",
- "session_expirations",
- "consecutive_session_expirations",
- "session_reauthentications_last_hour",
- "browser_disconnects",
- "consecutive_browser_disconnects",
- "browser_connect_failures",
- "consecutive_browser_connect_failures",
- "started_at",
- "updated_at",
- "exit_code",
- "reason",
- "calendar",
- }
- )
- class _SoakCancelled(Exception):
- """Internal control flow for a clean Ctrl+C."""
- class ItaSoakStatusError(ValueError):
- """Raised when the private local soak status cannot be written safely."""
- class ItaSoakBrowserConnectLimitError(RuntimeError):
- """Raised after two consecutive browser/CDP connection failures."""
- class ItaSoakSessionAttemptLimitError(RuntimeError):
- """Raised when an optional explicit Session-attempt limit is reached."""
- def calendar_soak_limits(env: Mapping[str, str]) -> dict[str, int]:
- """Parse the explicit Calendar experiment limits without reading secrets."""
- defaults = {
- "duration_seconds": 900,
- "min_wait": CALENDAR_MIN_WAIT_SECONDS,
- "max_wait": CALENDAR_MIN_WAIT_SECONDS,
- "max_session_attempts": DEFAULT_MAX_SESSION_ATTEMPTS,
- "restart_wait": DEFAULT_SESSION_RESTART_WAIT_SECONDS,
- "restart_after_probes": 0,
- }
- limits = {}
- for key, default in defaults.items():
- try:
- limits[key] = int(env.get(f"ITA_SOAK_{key.upper()}", str(default)))
- except (TypeError, ValueError):
- raise DebugConfigurationError(f"ITA_SOAK_{key.upper()} must be an integer") from None
- _validate_soak_limits(**{key: value for key, value in limits.items() if key != "restart_after_probes"})
- if limits["min_wait"] < CALENDAR_MIN_WAIT_SECONDS:
- raise DebugConfigurationError("Calendar soak min_wait must be at least 300 seconds")
- if limits["restart_after_probes"] < 0:
- raise DebugConfigurationError("restart_after_probes must be nonnegative")
- return limits
- class CalendarSoakContext:
- """Calendar-specific evidence, composed with the existing soak lifecycle.
- No scheduler, OTP reader, account pool, or publisher lives here. The owner
- transfers a single already-prepared plugin and audit to run_soak().
- """
- def __init__(self, *, plugin, route, request_audit, before):
- self.initial_plugin = plugin
- self.route = route
- self.config = plugin.config
- self.before = before
- self.audit = request_audit
- self.audit.enable_time_slot_counting()
- preparation = self.audit.settled_snapshot()
- if (preparation["otpRequests"] != 1 or preparation["avantiPosts"] != 1
- or preparation["finalRequests"] != 0):
- raise CalendarGateError("Calendar preparation exceeded the one-shot contract")
- self.preparation_counts = preparation
- plugin.free_config["query_mode"] = CALENDAR_QUERY_MODE
- self.config.free_config["query_mode"] = CALENDAR_QUERY_MODE
- self.checkpoint = plugin.capture_calendar_query_checkpoint(route.appointment_type())
- self.counts = {key: 0 for key in preparation}
- self.last_counts = preparation
- self.query_before = None
- self.previous_scan = None
- self.positive_queries = 0
- self.negative_queries = 0
- self.restores = 0
- self.audit_gaps = 0
- self._gap_recorded = False
- self.reservation_verified = False
- self.after = None
- self.queries_in_current_session = 0
- self.latest = None
- self.failure_reason = ""
- self.expiry_causes = {"configuredLifetime": 0, "authentication": 0}
- def _collect(self, *, allow_disconnected: bool = False):
- if self.audit is None:
- return None
- if self.audit.driver.is_running is not True:
- if not allow_disconnected:
- raise PageDisconnectedError("Calendar audit disconnected")
- if not self._gap_recorded:
- self.audit_gaps += 1
- self._gap_recorded = True
- current = self.audit.retained_snapshot()
- else:
- current = self.audit.settled_snapshot()
- if set(current) != set(self.last_counts):
- raise CalendarGateError("Calendar soak request counter shape changed")
- delta = {key: current[key] - self.last_counts[key] for key in current}
- if any(type(value) is not int or value < 0 for value in delta.values()):
- raise CalendarGateError("Calendar soak request counters moved backwards")
- for key, value in delta.items():
- self.counts[key] += value
- self.last_counts = current
- if any(self.counts[key] for key in ("otpRequests", "avantiPosts", "finalRequests")):
- raise CalendarGateError("Calendar soak observed a forbidden preparation/final request")
- return current
- def attach_restored_session(self, plugin):
- from tools.debug_ita_calendar_gate import CalendarRequestAudit
- self.audit = CalendarRequestAudit(plugin.page, include_time_slots=True)
- self.audit.start()
- self.last_counts = {key: 0 for key in self.counts}
- self._gap_recorded = False
- self.queries_in_current_session = 0
- self.latest = None
- plugin.restore_calendar_query(self.route.appointment_type(), self.checkpoint)
- self._collect()
- self.restores += 1
- print(f"ITA_SOAK_CALENDAR_RESTORED restores={self.restores} fresh_proof=true", flush=True)
- def begin_query(self, plugin):
- self.latest = None # A failed new probe must not expose an old date as current.
- self.failure_reason = ""
- self.previous_scan = plugin.last_calendar_query_scan
- self.query_before = self._collect()
- def record_query(self, plugin, result):
- from tools.debug_ita_calendar_gate import _query_request_delta, validate_production_query_result
- scan = plugin.last_calendar_query_scan
- if scan is self.previous_scan:
- raise CalendarGateError("Calendar soak reused old scan evidence")
- report = validate_production_query_result(
- result, scan, apt_type=self.route.appointment_type(),
- lookahead_months=self.checkpoint.route.calendar_scan.lookahead_months,
- )
- after = self._collect()
- standard_keys = {"otpRequests", "avantiPosts", "availabilityRequests", "finalRequests"}
- report["queryRequestDelta"] = _query_request_delta(
- {key: self.query_before[key] for key in standard_keys},
- {key: after[key] for key in standard_keys},
- expected_availability=scan.scanned_months,
- )
- report["automaticTimeSlots"] = after["timeSlotRequests"] - self.query_before["timeSlotRequests"]
- guard_calendar(plugin.page)
- if result.success:
- self.positive_queries += 1
- else:
- self.negative_queries += 1
- self.latest = report
- self.queries_in_current_session += 1
- print("ITA_SOAK_CALENDAR_QUERY_RESULT=" + json.dumps(report, sort_keys=True), flush=True)
- def close_session(self):
- if self.audit is None:
- return
- audit = self.audit
- try:
- self._collect(allow_disconnected=True)
- finally:
- try:
- audit.stop()
- finally:
- self.audit = None
- def verify_end(self, plugin):
- from tools.debug_ita_calendar_gate import read_reservation_snapshot
- if plugin is None or not plugin.page:
- return
- self._collect(allow_disconnected=True)
- if self.audit is None or self.audit.driver.is_running is not True:
- return
- after = read_reservation_snapshot(plugin.page)
- self.after = after
- if after != self.before:
- raise CalendarGateError("Calendar soak reservation or waitlist counts changed")
- self.reservation_verified = True
- def safe_dict(self):
- return {
- "mode": "calendar", "positiveQueries": self.positive_queries,
- "negativeQueries": self.negative_queries, "restores": self.restores,
- "queryRequests": dict(self.counts), "auditGaps": self.audit_gaps,
- "preparationRequests": dict(self.preparation_counts),
- "expiryCauses": dict(self.expiry_causes),
- "reservationVerified": self.reservation_verified,
- "reservationBefore": {"reservations": self.before.reservations, "waitlist": self.before.waitlist},
- "reservationAfter": (
- {"reservations": self.after.reservations, "waitlist": self.after.waitlist}
- if self.after is not None else None
- ),
- "queriesInCurrentSession": self.queries_in_current_session,
- "latestQuery": self.latest, "failureReason": self.failure_reason,
- "automaticOtpTodo": "ITA-OTP-01",
- }
- def run_prepared_calendar_soak(env, *, plugin, route, request_audit, before) -> int:
- """Take ownership from Q-CAL after exactly one manual preparation."""
- try:
- limits = calendar_soak_limits(env)
- context = CalendarSoakContext(
- plugin=plugin, route=route, request_audit=request_audit, before=before,
- )
- except Exception as exc:
- print(f"ITA_SOAK_CALENDAR_SETUP_FAILED type={type(exc).__name__}", flush=True)
- cleanup_failed = False
- for action in (request_audit.stop, plugin.cleanup):
- try:
- action()
- except Exception:
- cleanup_failed = True
- return 3 if cleanup_failed else 1
- return run_soak(
- env, apply=True, non_interactive=True, prepared_calendar=context, **limits,
- )
- def _ensure_private_directory(path: Path) -> Path:
- if path.is_symlink():
- raise ItaSoakStatusError("Italy soak status directory must not be a symlink")
- try:
- path.mkdir(mode=0o700, parents=True, exist_ok=True)
- except OSError as exc:
- raise ItaSoakStatusError(
- f"cannot create Italy soak status directory ({type(exc).__name__})"
- ) from exc
- if path.is_symlink() or not path.is_dir():
- raise ItaSoakStatusError(
- "Italy soak status path must be a private directory"
- )
- if os.name == "posix":
- try:
- path.chmod(0o700)
- mode = stat.S_IMODE(path.stat().st_mode)
- except OSError as exc:
- raise ItaSoakStatusError(
- f"cannot secure Italy soak status directory ({type(exc).__name__})"
- ) from exc
- if mode != 0o700:
- raise ItaSoakStatusError(
- "Italy soak status directory permissions must be 700"
- )
- return path.resolve()
- def write_soak_status(
- body: Mapping[str, object],
- *,
- status_path: str | Path = SOAK_STATUS_PATH,
- ) -> Path:
- """Atomically write a metadata-only local status snapshot."""
- unsupported = sorted(set(body) - _STATUS_KEYS)
- if unsupported:
- raise ItaSoakStatusError(
- "Italy soak status contains unsupported keys: "
- + ", ".join(unsupported)
- )
- path = Path(status_path).expanduser()
- if not path.is_absolute():
- path = REPO_ROOT / path
- directory = _ensure_private_directory(path.parent)
- resolved_path = directory / path.name
- encoded = json.dumps(
- dict(body),
- ensure_ascii=True,
- sort_keys=True,
- separators=(",", ":"),
- ).encode("utf-8")
- temporary = directory / (
- f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
- )
- descriptor: int | None = None
- try:
- flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
- flags |= getattr(os, "O_NOFOLLOW", 0)
- descriptor = os.open(temporary, flags, 0o600)
- if os.name == "posix":
- os.fchmod(descriptor, 0o600)
- with os.fdopen(descriptor, "wb") as handle:
- descriptor = None
- handle.write(encoded)
- handle.flush()
- os.fsync(handle.fileno())
- os.replace(temporary, resolved_path)
- if os.name == "posix":
- resolved_path.chmod(0o600)
- if stat.S_IMODE(resolved_path.stat().st_mode) != 0o600:
- raise ItaSoakStatusError(
- "Italy soak status file permissions must be 600"
- )
- except ItaSoakStatusError:
- raise
- except OSError as exc:
- raise ItaSoakStatusError(
- f"cannot write Italy soak status ({type(exc).__name__})"
- ) from exc
- finally:
- if descriptor is not None:
- os.close(descriptor)
- try:
- temporary.unlink()
- except FileNotFoundError:
- pass
- except OSError:
- pass
- return resolved_path
- def _validate_soak_limits(
- *,
- duration_seconds: int,
- min_wait: int,
- max_wait: int,
- max_session_attempts: int,
- restart_wait: int,
- ) -> None:
- if not MIN_DURATION_SECONDS <= duration_seconds <= DAY_SECONDS:
- raise DebugConfigurationError(
- f"duration_seconds must be between {MIN_DURATION_SECONDS} "
- f"and {DAY_SECONDS}"
- )
- if min_wait < DEFAULT_MIN_WAIT_SECONDS:
- raise DebugConfigurationError(
- f"min_wait must be at least {DEFAULT_MIN_WAIT_SECONDS} seconds"
- )
- if max_wait < min_wait:
- raise DebugConfigurationError("max_wait must be greater than min_wait")
- if max_session_attempts < 0:
- raise DebugConfigurationError(
- "max_session_attempts must be 0 (duration-bounded) or positive"
- )
- if restart_wait < DEFAULT_SESSION_RESTART_WAIT_SECONDS:
- raise DebugConfigurationError(
- "restart_wait must be at least "
- f"{DEFAULT_SESSION_RESTART_WAIT_SECONDS} seconds"
- )
- def _service_list_structure_for_log(plugin: ItaPlugin) -> str:
- raw = getattr(plugin, "last_service_list_structure", None)
- if not isinstance(raw, dict):
- return "not-available"
- allowed = {
- "path": raw.get("path"),
- "ready_state": raw.get("ready_state"),
- "table_count": int(raw.get("table_count") or 0),
- "row_count": int(raw.get("row_count") or 0),
- "booking_link_count": int(raw.get("booking_link_count") or 0),
- "target_link_count": int(raw.get("target_link_count") or 0),
- }
- return json.dumps(allowed, ensure_ascii=True, sort_keys=True)
- def run_soak(
- env: Mapping[str, str],
- *,
- apply: bool,
- non_interactive: bool,
- duration_seconds: int,
- min_wait: int,
- max_wait: int,
- max_session_attempts: int = DEFAULT_MAX_SESSION_ATTEMPTS,
- restart_wait: int = DEFAULT_SESSION_RESTART_WAIT_SECONDS,
- randint: Callable[[int, int], int] = random.randint,
- sleep: Callable[[float], None] = time.sleep,
- monotonic: Callable[[], float] = time.monotonic,
- wall_time: Callable[[], float] = time.time,
- plugin_factory: Callable[[str], ItaPlugin] | None = None,
- status_writer: Callable[[Mapping[str, object]], object] | None = None,
- prepared_calendar: CalendarSoakContext | None = None,
- restart_after_probes: int = 0,
- ) -> int:
- # The prepared instance owns its verified route. Do not reinterpret the
- # environment after ownership has transferred from the Calendar gate.
- route = prepared_calendar.route if prepared_calendar is not None else resolve_debug_route(env)
- route_log = (
- f"service_id={route.service_id} routing_key={route.routing_key}"
- )
- _validate_soak_limits(
- duration_seconds=duration_seconds,
- min_wait=min_wait,
- max_wait=max_wait,
- max_session_attempts=max_session_attempts,
- restart_wait=restart_wait,
- )
- if prepared_calendar is not None and min_wait < CALENDAR_MIN_WAIT_SECONDS:
- raise DebugConfigurationError("Calendar soak min_wait must be at least 300 seconds")
- if type(restart_after_probes) is not int or restart_after_probes < 0:
- raise DebugConfigurationError("restart_after_probes must be a nonnegative integer")
- if restart_after_probes and prepared_calendar is None:
- raise DebugConfigurationError("Controlled restart requires the prepared Calendar mode")
- if not apply:
- print(
- "ITA_SOAK_PREFLIGHT_OK "
- f"{route_log} "
- f"duration_seconds={duration_seconds} "
- f"min_wait={min_wait} max_wait={max_wait} "
- f"max_session_attempts={max_session_attempts} "
- "external_request=false"
- )
- return 0
- if prepared_calendar is None:
- chrome_path = resolve_chrome_path(env)
- os.environ["CHROME_BIN"] = str(chrome_path)
- username, password = _get_query_credentials(
- env,
- non_interactive=non_interactive,
- )
- profile_path = _prepare_query_profile(username)
- config = build_query_config(
- env,
- username=username,
- password=password,
- persistent_profile_path=profile_path,
- route=route,
- )
- else:
- config = prepared_calendar.config
- factory = plugin_factory or ItaPlugin
- writer = status_writer or write_soak_status
- apt_type = route.appointment_type()
- monitor_started = monotonic()
- started_wall = wall_time()
- plugin: ItaPlugin | None = prepared_calendar.initial_plugin if prepared_calendar else None
- current_session = 1 if prepared_calendar else 0
- probes = 0
- none_available = 0
- transient_errors = 0
- consecutive_transient_errors = 0
- session_attempts = current_session
- session_expirations = 0
- consecutive_session_expirations = 0
- session_reauthentication_times: list[float] = []
- browser_disconnects = 0
- consecutive_browser_disconnects = 0
- browser_connect_failures = 0
- consecutive_browser_connect_failures = 0
- exit_code = 0
- final_state = "running"
- final_reason = ""
- cleanup_error: Exception | None = None
- controlled_restart_done = False
- consecutive_calendar_network_errors = 0
- network_backoff = ExponentialBackoff(
- base_delay=restart_wait, max_delay=max_wait, jitter_range=(0, 0),
- )
- def elapsed_seconds() -> int:
- elapsed = max(0.0, monotonic() - monitor_started)
- return min(duration_seconds, int(elapsed))
- def status_body(state: str, *, code: int | None = None) -> dict:
- now = wall_time()
- current_monotonic = monotonic()
- reauthentications_last_hour = sum(
- 1
- for timestamp in session_reauthentication_times
- if 0
- <= current_monotonic - timestamp
- < SESSION_REAUTHENTICATION_WINDOW_SECONDS
- )
- body = {
- "state": state,
- "service_id": route.service_id,
- "routing_key": apt_type.routing_key,
- "duration_seconds": duration_seconds,
- "elapsed_seconds": elapsed_seconds(),
- "probes": probes,
- "none_available": none_available,
- "transient_errors": transient_errors,
- "consecutive_transient_errors": consecutive_transient_errors,
- "session_attempts": session_attempts,
- "session_expirations": session_expirations,
- "consecutive_session_expirations": (
- consecutive_session_expirations
- ),
- "session_reauthentications_last_hour": (
- reauthentications_last_hour
- ),
- "browser_disconnects": browser_disconnects,
- "consecutive_browser_disconnects": (
- consecutive_browser_disconnects
- ),
- "browser_connect_failures": browser_connect_failures,
- "consecutive_browser_connect_failures": (
- consecutive_browser_connect_failures
- ),
- "started_at": datetime.fromtimestamp(
- started_wall,
- tz=timezone.utc,
- ).isoformat(),
- "updated_at": datetime.fromtimestamp(
- now,
- tz=timezone.utc,
- ).isoformat(),
- }
- if code is not None:
- body["exit_code"] = code
- if final_reason:
- body["reason"] = final_reason
- if prepared_calendar is not None:
- body["calendar"] = prepared_calendar.safe_dict()
- return body
- def emit_status(state: str, *, code: int | None = None) -> None:
- writer(status_body(state, code=code))
- def cleanup_session(reason: str) -> None:
- nonlocal plugin, cleanup_error
- if plugin is None:
- return
- instance = plugin
- plugin = None
- try:
- try:
- if prepared_calendar is not None:
- prepared_calendar.close_session()
- finally:
- instance.cleanup()
- except Exception as exc:
- cleanup_error = exc
- raise
- print(
- "ITA_SOAK_SESSION_CLEANUP_OK "
- f"session={current_session} reason={reason}",
- flush=True,
- )
- def wait_before_session_restart() -> bool:
- remaining = duration_seconds - (monotonic() - monitor_started)
- wait_seconds = min(float(restart_wait), max(0.0, remaining))
- if wait_seconds:
- print(
- "ITA_SOAK_SESSION_RESTART_WAIT "
- f"seconds={int(wait_seconds)}",
- flush=True,
- )
- sleep(wait_seconds)
- return monotonic() - monitor_started < duration_seconds
- def start_session() -> bool:
- nonlocal plugin
- nonlocal current_session
- nonlocal session_attempts
- nonlocal transient_errors
- nonlocal browser_connect_failures
- nonlocal consecutive_browser_connect_failures
- while monotonic() - monitor_started < duration_seconds:
- if (
- max_session_attempts > 0
- and session_attempts >= max_session_attempts
- ):
- raise ItaSoakSessionAttemptLimitError
- session_attempts += 1
- current_session = session_attempts
- instance = factory("ita-soak-local-debug")
- plugin = instance
- instance.set_log(lambda message: print(message, flush=True))
- instance.set_config(config)
- try:
- instance.create_session()
- except BrowserConnectError:
- browser_connect_failures += 1
- transient_errors += 1
- consecutive_browser_connect_failures += 1
- print(
- "ITA_SOAK_BROWSER_CONNECT_FAILED "
- f"session_attempt={current_session} "
- f"consecutive={consecutive_browser_connect_failures} "
- f"total={browser_connect_failures}",
- flush=True,
- )
- cleanup_session("browser-connect-failed")
- if (
- consecutive_browser_connect_failures
- >= MAX_CONSECUTIVE_BROWSER_CONNECT_FAILURES
- ):
- raise ItaSoakBrowserConnectLimitError
- if not wait_before_session_restart():
- return False
- continue
- consecutive_browser_connect_failures = 0
- if prepared_calendar is not None:
- prepared_calendar.attach_restored_session(instance)
- print(
- "ITA_SOAK_SESSION_STARTED "
- f"session={current_session} "
- f"session_attempts={session_attempts}",
- flush=True,
- )
- return True
- return False
- try:
- if plugin is None:
- start_session()
- print(
- "ITA_SOAK_STARTED "
- f"{route_log} "
- f"duration_seconds={duration_seconds} "
- f"min_wait={min_wait} max_wait={max_wait}",
- flush=True,
- )
- emit_status("running")
- while True:
- if monotonic() - monitor_started >= duration_seconds:
- final_state = "completed"
- print(
- "ITA_SOAK_COMPLETED "
- f"duration_seconds={duration_seconds} probes={probes} "
- f"none_available={none_available} "
- f"transient_errors={transient_errors} "
- f"session_attempts={session_attempts}",
- flush=True,
- )
- break
- if (prepared_calendar is not None and restart_after_probes
- and not controlled_restart_done
- and prepared_calendar.positive_queries + prepared_calendar.negative_queries >= restart_after_probes):
- # A deliberate normal close tests ownership and restore, not
- # a fabricated website outage or an authentication expiry.
- controlled_restart_done = True
- emit_status("controlled_restart")
- cleanup_session("controlled-restart")
- if not wait_before_session_restart():
- continue
- start_session()
- emit_status("running")
- continue
- probes += 1
- print(
- "ITA_SOAK_PROBE_START "
- f"probe={probes} session={current_session} "
- f"elapsed_seconds={elapsed_seconds()}",
- flush=True,
- )
- try:
- if prepared_calendar is not None:
- if not plugin.health_check():
- if getattr(plugin, "_query_stop_reason", None) == "browser_disconnected":
- raise PageDisconnectedError("Calendar browser disconnected")
- raise SessionExpiredOrInvalidError("Calendar Session health expired")
- prepared_calendar.begin_query(plugin)
- result = plugin.query(apt_type)
- if prepared_calendar is not None:
- prepared_calendar.record_query(plugin, result)
- except ItaCalendarTransientError:
- if prepared_calendar is None:
- raise
- prepared_calendar.latest = None
- transient_errors += 1
- consecutive_calendar_network_errors += 1
- prepared_calendar.failure_reason = "transient-network"
- emit_status("cooldown")
- if consecutive_calendar_network_errors >= 2:
- final_state, final_reason, exit_code = "failed", "consecutive-calendar-network-errors", 1
- break
- remaining = duration_seconds - (monotonic() - monitor_started)
- sleep(min(network_backoff.calculate(consecutive_calendar_network_errors), max(0.0, remaining)))
- continue
- except ItaTargetServiceLinkMissingError:
- consecutive_session_expirations = 0
- consecutive_browser_disconnects = 0
- transient_errors += 1
- consecutive_transient_errors += 1
- print(
- "ITA_SERVICE_LIST_STRUCTURE="
- + _service_list_structure_for_log(plugin),
- flush=True,
- )
- print(
- "ITA_SOAK_TRANSIENT "
- "type=ItaTargetServiceLinkMissingError "
- f"consecutive={consecutive_transient_errors} "
- f"total={transient_errors}",
- flush=True,
- )
- if (
- consecutive_transient_errors
- >= MAX_CONSECUTIVE_TARGET_LINK_MISSES
- ):
- final_state = "failed"
- final_reason = "consecutive-target-link-misses"
- exit_code = 1
- print(
- "ITA_SOAK_FAILED "
- "reason=consecutive-target-link-misses",
- flush=True,
- )
- break
- except PageDisconnectedError:
- if prepared_calendar is not None:
- prepared_calendar.latest = None
- prepared_calendar.failure_reason = "browser-disconnected"
- browser_disconnects += 1
- transient_errors += 1
- consecutive_browser_disconnects += 1
- consecutive_transient_errors = 0
- print(
- "ITA_SOAK_BROWSER_DISCONNECTED "
- f"session={current_session} "
- f"consecutive={consecutive_browser_disconnects} "
- f"total={browser_disconnects}",
- flush=True,
- )
- cleanup_session("browser-disconnected")
- if (
- consecutive_browser_disconnects
- >= MAX_CONSECUTIVE_BROWSER_DISCONNECTS
- ):
- final_state = "failed"
- final_reason = "consecutive-browser-disconnects"
- exit_code = 1
- print(
- "ITA_SOAK_FAILED "
- "reason=consecutive-browser-disconnects",
- flush=True,
- )
- break
- if not wait_before_session_restart():
- continue
- start_session()
- emit_status("running")
- continue
- except SessionExpiredOrInvalidError:
- if prepared_calendar is not None:
- prepared_calendar.latest = None
- prepared_calendar.failure_reason = "session-expired"
- expiry_cause = (
- "configuredLifetime"
- if getattr(plugin, "_query_stop_reason", None) == "configured_lifetime_reached"
- else "authentication"
- )
- prepared_calendar.expiry_causes[expiry_cause] += 1
- session_expirations += 1
- consecutive_session_expirations += 1
- consecutive_transient_errors = 0
- consecutive_browser_disconnects = 0
- current_monotonic = monotonic()
- session_reauthentication_times[:] = [
- timestamp
- for timestamp in session_reauthentication_times
- if 0
- <= current_monotonic - timestamp
- < SESSION_REAUTHENTICATION_WINDOW_SECONDS
- ]
- session_reauthentication_times.append(current_monotonic)
- reauthentications_last_hour = len(
- session_reauthentication_times
- )
- print(
- "ITA_SOAK_SESSION_EXPIRED "
- f"session={current_session} "
- f"session_expirations={session_expirations} "
- f"consecutive={consecutive_session_expirations} "
- f"reauthentications_last_hour="
- f"{reauthentications_last_hour}",
- flush=True,
- )
- cleanup_session("session-expired")
- if (
- consecutive_session_expirations
- >= MAX_CONSECUTIVE_SESSION_EXPIRATIONS
- ):
- final_state = "failed"
- final_reason = "consecutive-session-expirations"
- exit_code = 1
- print(
- "ITA_SOAK_FAILED "
- "reason=consecutive-session-expirations",
- flush=True,
- )
- break
- if (
- reauthentications_last_hour
- > MAX_SESSION_REAUTHENTICATIONS_PER_WINDOW
- ):
- final_state = "failed"
- final_reason = "reauthentication-rate-limit"
- exit_code = 1
- print(
- "ITA_SOAK_FAILED "
- "reason=reauthentication-rate-limit",
- flush=True,
- )
- break
- if not wait_before_session_restart():
- continue
- start_session()
- emit_status("running")
- continue
- else:
- consecutive_calendar_network_errors = 0
- consecutive_transient_errors = 0
- consecutive_session_expirations = 0
- consecutive_browser_disconnects = 0
- if prepared_calendar is not None:
- if not result.success:
- none_available += 1
- elif _is_weak_entry_signal(result):
- _print_booking_form_structure(plugin, route=route)
- final_state = "weak_signal"
- final_reason = "booking-form"
- exit_code = 4
- print(
- "\aITA_SOAK_WEAK_SIGNAL "
- f"{route_log} probe={probes} "
- "evidence=booking-form",
- flush=True,
- )
- break
- elif _is_nonempty_query_result(result):
- final_state = "safe_stop"
- final_reason = "unexpected-non-empty-result"
- exit_code = 4
- print(
- "ITA_SOAK_SAFE_STOP "
- "reason=unexpected-non-empty-result",
- flush=True,
- )
- break
- if prepared_calendar is None:
- none_available += 1
- print(
- "ITA_SOAK_NONE_AVAILABLE "
- f"probe={probes} session={current_session}",
- flush=True,
- )
- emit_status("running")
- remaining = duration_seconds - (monotonic() - monitor_started)
- if remaining <= 0:
- continue
- wait_seconds = min(float(randint(min_wait, max_wait)), remaining)
- print(
- "ITA_SOAK_WAIT "
- f"next_probe={probes + 1} seconds={int(wait_seconds)}",
- flush=True,
- )
- sleep(wait_seconds)
- except ItaSoakBrowserConnectLimitError:
- final_state = "failed"
- final_reason = "consecutive-browser-connect-failures"
- exit_code = 1
- print(
- "ITA_SOAK_FAILED "
- "reason=consecutive-browser-connect-failures",
- flush=True,
- )
- except ItaSoakSessionAttemptLimitError:
- final_state = "failed"
- final_reason = "max-session-attempts-reached"
- exit_code = 1
- print(
- "ITA_SOAK_FAILED reason=max-session-attempts-reached",
- flush=True,
- )
- except ItaCalendarPreparationRequiredError:
- final_state, final_reason, exit_code = "needs_manual", "calendar-preparation-required", 4
- if prepared_calendar is not None:
- prepared_calendar.latest = None
- prepared_calendar.failure_reason = final_reason
- print("ITA_SOAK_NEEDS_MANUAL todo=ITA-OTP-01 otp_sent=false avanti_submitted=false", flush=True)
- except (PermissionDeniedError, ItaAuthenticationChallengeError):
- final_state, final_reason, exit_code = "needs_manual", "website-verification-required", 4
- print("ITA_SOAK_NEEDS_MANUAL reason=website-verification-required", flush=True)
- except RateLimiteddError:
- final_state, final_reason, exit_code = "rate_limited", "server-rate-limit", 5
- print("ITA_SOAK_RATE_LIMITED retry=false account_rotation=false", flush=True)
- except KeyboardInterrupt:
- final_state = "cancelled"
- final_reason = "operator-cancelled"
- exit_code = 130
- print(
- "\nITA_SOAK_CANCELLED reason=operator-cancelled",
- flush=True,
- )
- except (BizLogicError, CalendarGateError) as exc:
- final_state = "failed"
- final_reason = "business-error"
- exit_code = 1
- print(
- f"ITA_SOAK_FAILED type={type(exc).__name__} "
- f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(exc)}",
- flush=True,
- )
- if plugin is not None and prepared_calendar is None:
- _print_failure_context(plugin)
- except Exception as exc:
- final_state = "failed"
- final_reason = "unexpected-error"
- exit_code = 1
- print(
- f"ITA_SOAK_FAILED type={type(exc).__name__} "
- f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(exc)}",
- flush=True,
- )
- if plugin is not None and prepared_calendar is None:
- _print_failure_context(plugin)
- finally:
- if prepared_calendar is not None:
- if exit_code != 0:
- prepared_calendar.latest = None
- prepared_calendar.failure_reason = final_reason
- try:
- prepared_calendar.verify_end(plugin)
- except Exception as exc:
- print(f"ITA_SOAK_END_EVIDENCE_UNAVAILABLE type={type(exc).__name__}", flush=True)
- if exit_code == 0:
- has_queries = prepared_calendar.positive_queries + prepared_calendar.negative_queries > 0
- if (not has_queries or prepared_calendar.audit_gaps
- or not prepared_calendar.reservation_verified
- or prepared_calendar.queries_in_current_session == 0
- or consecutive_calendar_network_errors
- or consecutive_session_expirations or consecutive_browser_disconnects
- or (restart_after_probes and not controlled_restart_done)):
- final_state, final_reason, exit_code = "incomplete", "calendar-evidence-incomplete", 6
- try:
- cleanup_session("soak-finished")
- except Exception as exc:
- cleanup_error = exc
- if cleanup_error is not None:
- final_state = "failed"
- final_reason = "cleanup-failed"
- print(
- f"ITA_SOAK_CLEANUP_FAILED type={type(cleanup_error).__name__} "
- f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(cleanup_error)}",
- file=sys.stderr,
- flush=True,
- )
- else:
- print("ITA_SOAK_CLEANUP_OK", flush=True)
- final_code = 3 if cleanup_error is not None else exit_code
- if prepared_calendar is not None and final_code != 0:
- prepared_calendar.failure_reason = final_reason
- prepared_calendar.latest = None
- try:
- emit_status(final_state, code=final_code)
- except Exception as exc:
- cleanup_error = cleanup_error or exc
- if prepared_calendar is not None:
- final_state, final_reason = "failed", "status-write-failed"
- prepared_calendar.failure_reason = final_reason
- print(
- f"ITA_SOAK_STATUS_FAILED type={type(exc).__name__}",
- file=sys.stderr,
- flush=True,
- )
- if prepared_calendar is not None:
- report = prepared_calendar.safe_dict()
- report.update({
- "state": final_state, "reason": final_reason,
- "elapsedSeconds": max(0, int(monotonic() - monitor_started)), "queryAttempts": probes,
- "sessionAttempts": session_attempts,
- "controlledRestart": controlled_restart_done,
- "exitCode": 3 if cleanup_error is not None else exit_code,
- "passed": cleanup_error is None and exit_code == 0,
- })
- print("ITA_SOAK_CALENDAR_RESULT=" + json.dumps(report, sort_keys=True), flush=True)
- return 3 if cleanup_error is not None else exit_code
- def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
- parser = argparse.ArgumentParser(
- description=(
- "Duration-bounded route-configured Italy Query soak. Without --run, "
- "performs a zero-network preflight only."
- )
- )
- parser.add_argument(
- "--calendar",
- action="store_true",
- help="Use one manual Calendar preparation, then the production Query and read-only recovery.",
- )
- parser.add_argument(
- "--run",
- action="store_true",
- help="Start the real website soak.",
- )
- parser.add_argument(
- "--non-interactive",
- action="store_true",
- help="Fail instead of prompting when local credentials are unavailable.",
- )
- parser.add_argument(
- "--duration-seconds",
- type=int,
- default=None,
- help=f"Soak duration in seconds (maximum: {DAY_SECONDS}).",
- )
- parser.add_argument(
- "--min-wait",
- type=int,
- default=None,
- help=f"Minimum seconds between probes (Calendar floor: {CALENDAR_MIN_WAIT_SECONDS}; entry-probe floor: {DEFAULT_MIN_WAIT_SECONDS}).",
- )
- parser.add_argument(
- "--max-wait",
- type=int,
- default=None,
- help="Maximum seconds between probes.",
- )
- parser.add_argument(
- "--max-session-attempts",
- type=int,
- default=DEFAULT_MAX_SESSION_ATTEMPTS,
- help=(
- "Optional total Session-attempt limit; 0 uses the duration and "
- "reauthentication circuit breakers only."
- ),
- )
- parser.add_argument(
- "--restart-wait",
- type=int,
- default=DEFAULT_SESSION_RESTART_WAIT_SECONDS,
- help=(
- "Minimum seconds before recreating an explicitly expired Session "
- f"(floor: {DEFAULT_SESSION_RESTART_WAIT_SECONDS})."
- ),
- )
- parser.add_argument(
- "--restart-after-probes", type=int, default=0,
- help="Calendar only: normally close/recreate once after this many successful queries (0 disables).",
- )
- return parser.parse_args(argv)
- def main(argv: list[str] | None = None) -> int:
- args = parse_args(argv)
- try:
- if args.calendar:
- from tools.debug_ita_calendar_gate import build_calendar_gate_config, run_calendar_gate
- env = dict(os.environ)
- for key in (
- "duration_seconds", "min_wait", "max_wait", "max_session_attempts",
- "restart_wait", "restart_after_probes",
- ):
- value = getattr(args, key)
- if value is not None:
- env[f"ITA_SOAK_{key.upper()}"] = str(value)
- limits = calendar_soak_limits(env)
- # Configuration only: no credential lookup, profile creation, or login.
- build_calendar_gate_config(
- env, username="preflight@example.invalid", password="unused-preflight",
- )
- if not args.run:
- print("ITA_SOAK_CALENDAR_PREFLIGHT_OK " + json.dumps(limits, sort_keys=True)
- + " external_request=false", flush=True)
- return 0
- if args.non_interactive:
- raise DebugConfigurationError("Calendar preparation currently requires manual OTP input")
- env["ITA_QCAL_MODE"] = "calendar_soak"
- return run_calendar_gate(env)
- if args.restart_after_probes:
- raise DebugConfigurationError("--restart-after-probes requires --calendar")
- return run_soak(
- os.environ,
- apply=args.run,
- non_interactive=args.non_interactive,
- duration_seconds=args.duration_seconds if args.duration_seconds is not None else DAY_SECONDS,
- min_wait=args.min_wait if args.min_wait is not None else DEFAULT_MIN_WAIT_SECONDS,
- max_wait=args.max_wait if args.max_wait is not None else DEFAULT_MAX_WAIT_SECONDS,
- max_session_attempts=args.max_session_attempts,
- restart_wait=args.restart_wait,
- )
- except (DebugConfigurationError, CalendarGateError) as exc:
- print(f"ITA_SOAK_CONFIG_ERROR: {exc}", file=sys.stderr)
- return 2
- if __name__ == "__main__":
- raise SystemExit(main())
|