debug_ita_soak.py 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170
  1. #!/usr/bin/env python3
  2. """Run a duration-bounded local Italy Query soak with no booking actions."""
  3. from __future__ import annotations
  4. import argparse
  5. import json
  6. import os
  7. import random
  8. import stat
  9. import sys
  10. import time
  11. import uuid
  12. from datetime import datetime, timezone
  13. from pathlib import Path
  14. from typing import Callable, Mapping
  15. from DrissionPage.errors import BrowserConnectError, PageDisconnectedError
  16. REPO_ROOT = Path(__file__).resolve().parents[1]
  17. if str(REPO_ROOT) not in sys.path:
  18. sys.path.insert(0, str(REPO_ROOT))
  19. from plugins.ita_plugin import (
  20. CALENDAR_QUERY_MODE,
  21. ItaAuthenticationChallengeError,
  22. ItaCalendarPreparationRequiredError,
  23. ItaCalendarTransientError,
  24. ItaPlugin,
  25. ItaTargetServiceLinkMissingError,
  26. )
  27. from tools.debug_ita_query import (
  28. _get_query_credentials,
  29. _is_nonempty_query_result,
  30. _is_weak_entry_signal,
  31. _popup_text_for_log,
  32. _prepare_query_profile,
  33. _print_booking_form_structure,
  34. _print_failure_context,
  35. build_query_config,
  36. resolve_debug_route,
  37. )
  38. from tools.debug_ita_session import (
  39. DebugConfigurationError,
  40. resolve_chrome_path,
  41. )
  42. from plugins.ita_plugin import CalendarGateError, guard_calendar
  43. from toolkit.backoff import ExponentialBackoff
  44. from vs_types import BizLogicError, PermissionDeniedError, RateLimiteddError, SessionExpiredOrInvalidError
  45. DAY_SECONDS = 24 * 60 * 60
  46. MIN_DURATION_SECONDS = 60
  47. DEFAULT_MIN_WAIT_SECONDS = 60
  48. DEFAULT_MAX_WAIT_SECONDS = 300
  49. DEFAULT_MAX_SESSION_ATTEMPTS = 0
  50. DEFAULT_SESSION_RESTART_WAIT_SECONDS = 60
  51. MAX_CONSECUTIVE_TARGET_LINK_MISSES = 2
  52. MAX_CONSECUTIVE_BROWSER_DISCONNECTS = 2
  53. MAX_CONSECUTIVE_SESSION_EXPIRATIONS = 2
  54. MAX_CONSECUTIVE_BROWSER_CONNECT_FAILURES = 2
  55. SESSION_REAUTHENTICATION_WINDOW_SECONDS = 60 * 60
  56. MAX_SESSION_REAUTHENTICATIONS_PER_WINDOW = 6
  57. SOAK_STATUS_PATH = REPO_ROOT / "data" / "ita_soak" / "status.json"
  58. CALENDAR_MIN_WAIT_SECONDS = 300
  59. _STATUS_KEYS = frozenset(
  60. {
  61. "state",
  62. "service_id",
  63. "routing_key",
  64. "duration_seconds",
  65. "elapsed_seconds",
  66. "probes",
  67. "none_available",
  68. "transient_errors",
  69. "consecutive_transient_errors",
  70. "session_attempts",
  71. "session_expirations",
  72. "consecutive_session_expirations",
  73. "session_reauthentications_last_hour",
  74. "browser_disconnects",
  75. "consecutive_browser_disconnects",
  76. "browser_connect_failures",
  77. "consecutive_browser_connect_failures",
  78. "started_at",
  79. "updated_at",
  80. "exit_code",
  81. "reason",
  82. "calendar",
  83. }
  84. )
  85. class _SoakCancelled(Exception):
  86. """Internal control flow for a clean Ctrl+C."""
  87. class ItaSoakStatusError(ValueError):
  88. """Raised when the private local soak status cannot be written safely."""
  89. class ItaSoakBrowserConnectLimitError(RuntimeError):
  90. """Raised after two consecutive browser/CDP connection failures."""
  91. class ItaSoakSessionAttemptLimitError(RuntimeError):
  92. """Raised when an optional explicit Session-attempt limit is reached."""
  93. def calendar_soak_limits(env: Mapping[str, str]) -> dict[str, int]:
  94. """Parse the explicit Calendar experiment limits without reading secrets."""
  95. defaults = {
  96. "duration_seconds": 900,
  97. "min_wait": CALENDAR_MIN_WAIT_SECONDS,
  98. "max_wait": CALENDAR_MIN_WAIT_SECONDS,
  99. "max_session_attempts": DEFAULT_MAX_SESSION_ATTEMPTS,
  100. "restart_wait": DEFAULT_SESSION_RESTART_WAIT_SECONDS,
  101. "restart_after_probes": 0,
  102. }
  103. limits = {}
  104. for key, default in defaults.items():
  105. try:
  106. limits[key] = int(env.get(f"ITA_SOAK_{key.upper()}", str(default)))
  107. except (TypeError, ValueError):
  108. raise DebugConfigurationError(f"ITA_SOAK_{key.upper()} must be an integer") from None
  109. _validate_soak_limits(**{key: value for key, value in limits.items() if key != "restart_after_probes"})
  110. if limits["min_wait"] < CALENDAR_MIN_WAIT_SECONDS:
  111. raise DebugConfigurationError("Calendar soak min_wait must be at least 300 seconds")
  112. if limits["restart_after_probes"] < 0:
  113. raise DebugConfigurationError("restart_after_probes must be nonnegative")
  114. return limits
  115. class CalendarSoakContext:
  116. """Calendar-specific evidence, composed with the existing soak lifecycle.
  117. No scheduler, OTP reader, account pool, or publisher lives here. The owner
  118. transfers a single already-prepared plugin and audit to run_soak().
  119. """
  120. def __init__(self, *, plugin, route, request_audit, before):
  121. self.initial_plugin = plugin
  122. self.route = route
  123. self.config = plugin.config
  124. self.before = before
  125. self.audit = request_audit
  126. self.audit.enable_time_slot_counting()
  127. preparation = self.audit.settled_snapshot()
  128. if (preparation["otpRequests"] != 1 or preparation["avantiPosts"] != 1
  129. or preparation["finalRequests"] != 0):
  130. raise CalendarGateError("Calendar preparation exceeded the one-shot contract")
  131. self.preparation_counts = preparation
  132. plugin.free_config["query_mode"] = CALENDAR_QUERY_MODE
  133. self.config.free_config["query_mode"] = CALENDAR_QUERY_MODE
  134. self.checkpoint = plugin.capture_calendar_query_checkpoint(route.appointment_type())
  135. self.counts = {key: 0 for key in preparation}
  136. self.last_counts = preparation
  137. self.query_before = None
  138. self.previous_scan = None
  139. self.positive_queries = 0
  140. self.negative_queries = 0
  141. self.restores = 0
  142. self.audit_gaps = 0
  143. self._gap_recorded = False
  144. self.reservation_verified = False
  145. self.after = None
  146. self.queries_in_current_session = 0
  147. self.latest = None
  148. self.failure_reason = ""
  149. self.expiry_causes = {"configuredLifetime": 0, "authentication": 0}
  150. def _collect(self, *, allow_disconnected: bool = False):
  151. if self.audit is None:
  152. return None
  153. if self.audit.driver.is_running is not True:
  154. if not allow_disconnected:
  155. raise PageDisconnectedError("Calendar audit disconnected")
  156. if not self._gap_recorded:
  157. self.audit_gaps += 1
  158. self._gap_recorded = True
  159. current = self.audit.retained_snapshot()
  160. else:
  161. current = self.audit.settled_snapshot()
  162. if set(current) != set(self.last_counts):
  163. raise CalendarGateError("Calendar soak request counter shape changed")
  164. delta = {key: current[key] - self.last_counts[key] for key in current}
  165. if any(type(value) is not int or value < 0 for value in delta.values()):
  166. raise CalendarGateError("Calendar soak request counters moved backwards")
  167. for key, value in delta.items():
  168. self.counts[key] += value
  169. self.last_counts = current
  170. if any(self.counts[key] for key in ("otpRequests", "avantiPosts", "finalRequests")):
  171. raise CalendarGateError("Calendar soak observed a forbidden preparation/final request")
  172. return current
  173. def attach_restored_session(self, plugin):
  174. from tools.debug_ita_calendar_gate import CalendarRequestAudit
  175. self.audit = CalendarRequestAudit(plugin.page, include_time_slots=True)
  176. self.audit.start()
  177. self.last_counts = {key: 0 for key in self.counts}
  178. self._gap_recorded = False
  179. self.queries_in_current_session = 0
  180. self.latest = None
  181. plugin.restore_calendar_query(self.route.appointment_type(), self.checkpoint)
  182. self._collect()
  183. self.restores += 1
  184. print(f"ITA_SOAK_CALENDAR_RESTORED restores={self.restores} fresh_proof=true", flush=True)
  185. def begin_query(self, plugin):
  186. self.latest = None # A failed new probe must not expose an old date as current.
  187. self.failure_reason = ""
  188. self.previous_scan = plugin.last_calendar_query_scan
  189. self.query_before = self._collect()
  190. def record_query(self, plugin, result):
  191. from tools.debug_ita_calendar_gate import _query_request_delta, validate_production_query_result
  192. scan = plugin.last_calendar_query_scan
  193. if scan is self.previous_scan:
  194. raise CalendarGateError("Calendar soak reused old scan evidence")
  195. report = validate_production_query_result(
  196. result, scan, apt_type=self.route.appointment_type(),
  197. lookahead_months=self.checkpoint.route.calendar_scan.lookahead_months,
  198. )
  199. after = self._collect()
  200. standard_keys = {"otpRequests", "avantiPosts", "availabilityRequests", "finalRequests"}
  201. report["queryRequestDelta"] = _query_request_delta(
  202. {key: self.query_before[key] for key in standard_keys},
  203. {key: after[key] for key in standard_keys},
  204. expected_availability=scan.scanned_months,
  205. )
  206. report["automaticTimeSlots"] = after["timeSlotRequests"] - self.query_before["timeSlotRequests"]
  207. guard_calendar(plugin.page)
  208. if result.success:
  209. self.positive_queries += 1
  210. else:
  211. self.negative_queries += 1
  212. self.latest = report
  213. self.queries_in_current_session += 1
  214. print("ITA_SOAK_CALENDAR_QUERY_RESULT=" + json.dumps(report, sort_keys=True), flush=True)
  215. def close_session(self):
  216. if self.audit is None:
  217. return
  218. audit = self.audit
  219. try:
  220. self._collect(allow_disconnected=True)
  221. finally:
  222. try:
  223. audit.stop()
  224. finally:
  225. self.audit = None
  226. def verify_end(self, plugin):
  227. from tools.debug_ita_calendar_gate import read_reservation_snapshot
  228. if plugin is None or not plugin.page:
  229. return
  230. self._collect(allow_disconnected=True)
  231. if self.audit is None or self.audit.driver.is_running is not True:
  232. return
  233. after = read_reservation_snapshot(plugin.page)
  234. self.after = after
  235. if after != self.before:
  236. raise CalendarGateError("Calendar soak reservation or waitlist counts changed")
  237. self.reservation_verified = True
  238. def safe_dict(self):
  239. return {
  240. "mode": "calendar", "positiveQueries": self.positive_queries,
  241. "negativeQueries": self.negative_queries, "restores": self.restores,
  242. "queryRequests": dict(self.counts), "auditGaps": self.audit_gaps,
  243. "preparationRequests": dict(self.preparation_counts),
  244. "expiryCauses": dict(self.expiry_causes),
  245. "reservationVerified": self.reservation_verified,
  246. "reservationBefore": {"reservations": self.before.reservations, "waitlist": self.before.waitlist},
  247. "reservationAfter": (
  248. {"reservations": self.after.reservations, "waitlist": self.after.waitlist}
  249. if self.after is not None else None
  250. ),
  251. "queriesInCurrentSession": self.queries_in_current_session,
  252. "latestQuery": self.latest, "failureReason": self.failure_reason,
  253. "automaticOtpTodo": "ITA-OTP-01",
  254. }
  255. def run_prepared_calendar_soak(env, *, plugin, route, request_audit, before) -> int:
  256. """Take ownership from Q-CAL after exactly one manual preparation."""
  257. try:
  258. limits = calendar_soak_limits(env)
  259. context = CalendarSoakContext(
  260. plugin=plugin, route=route, request_audit=request_audit, before=before,
  261. )
  262. except Exception as exc:
  263. print(f"ITA_SOAK_CALENDAR_SETUP_FAILED type={type(exc).__name__}", flush=True)
  264. cleanup_failed = False
  265. for action in (request_audit.stop, plugin.cleanup):
  266. try:
  267. action()
  268. except Exception:
  269. cleanup_failed = True
  270. return 3 if cleanup_failed else 1
  271. return run_soak(
  272. env, apply=True, non_interactive=True, prepared_calendar=context, **limits,
  273. )
  274. def _ensure_private_directory(path: Path) -> Path:
  275. if path.is_symlink():
  276. raise ItaSoakStatusError("Italy soak status directory must not be a symlink")
  277. try:
  278. path.mkdir(mode=0o700, parents=True, exist_ok=True)
  279. except OSError as exc:
  280. raise ItaSoakStatusError(
  281. f"cannot create Italy soak status directory ({type(exc).__name__})"
  282. ) from exc
  283. if path.is_symlink() or not path.is_dir():
  284. raise ItaSoakStatusError(
  285. "Italy soak status path must be a private directory"
  286. )
  287. if os.name == "posix":
  288. try:
  289. path.chmod(0o700)
  290. mode = stat.S_IMODE(path.stat().st_mode)
  291. except OSError as exc:
  292. raise ItaSoakStatusError(
  293. f"cannot secure Italy soak status directory ({type(exc).__name__})"
  294. ) from exc
  295. if mode != 0o700:
  296. raise ItaSoakStatusError(
  297. "Italy soak status directory permissions must be 700"
  298. )
  299. return path.resolve()
  300. def write_soak_status(
  301. body: Mapping[str, object],
  302. *,
  303. status_path: str | Path = SOAK_STATUS_PATH,
  304. ) -> Path:
  305. """Atomically write a metadata-only local status snapshot."""
  306. unsupported = sorted(set(body) - _STATUS_KEYS)
  307. if unsupported:
  308. raise ItaSoakStatusError(
  309. "Italy soak status contains unsupported keys: "
  310. + ", ".join(unsupported)
  311. )
  312. path = Path(status_path).expanduser()
  313. if not path.is_absolute():
  314. path = REPO_ROOT / path
  315. directory = _ensure_private_directory(path.parent)
  316. resolved_path = directory / path.name
  317. encoded = json.dumps(
  318. dict(body),
  319. ensure_ascii=True,
  320. sort_keys=True,
  321. separators=(",", ":"),
  322. ).encode("utf-8")
  323. temporary = directory / (
  324. f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
  325. )
  326. descriptor: int | None = None
  327. try:
  328. flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
  329. flags |= getattr(os, "O_NOFOLLOW", 0)
  330. descriptor = os.open(temporary, flags, 0o600)
  331. if os.name == "posix":
  332. os.fchmod(descriptor, 0o600)
  333. with os.fdopen(descriptor, "wb") as handle:
  334. descriptor = None
  335. handle.write(encoded)
  336. handle.flush()
  337. os.fsync(handle.fileno())
  338. os.replace(temporary, resolved_path)
  339. if os.name == "posix":
  340. resolved_path.chmod(0o600)
  341. if stat.S_IMODE(resolved_path.stat().st_mode) != 0o600:
  342. raise ItaSoakStatusError(
  343. "Italy soak status file permissions must be 600"
  344. )
  345. except ItaSoakStatusError:
  346. raise
  347. except OSError as exc:
  348. raise ItaSoakStatusError(
  349. f"cannot write Italy soak status ({type(exc).__name__})"
  350. ) from exc
  351. finally:
  352. if descriptor is not None:
  353. os.close(descriptor)
  354. try:
  355. temporary.unlink()
  356. except FileNotFoundError:
  357. pass
  358. except OSError:
  359. pass
  360. return resolved_path
  361. def _validate_soak_limits(
  362. *,
  363. duration_seconds: int,
  364. min_wait: int,
  365. max_wait: int,
  366. max_session_attempts: int,
  367. restart_wait: int,
  368. ) -> None:
  369. if not MIN_DURATION_SECONDS <= duration_seconds <= DAY_SECONDS:
  370. raise DebugConfigurationError(
  371. f"duration_seconds must be between {MIN_DURATION_SECONDS} "
  372. f"and {DAY_SECONDS}"
  373. )
  374. if min_wait < DEFAULT_MIN_WAIT_SECONDS:
  375. raise DebugConfigurationError(
  376. f"min_wait must be at least {DEFAULT_MIN_WAIT_SECONDS} seconds"
  377. )
  378. if max_wait < min_wait:
  379. raise DebugConfigurationError("max_wait must be greater than min_wait")
  380. if max_session_attempts < 0:
  381. raise DebugConfigurationError(
  382. "max_session_attempts must be 0 (duration-bounded) or positive"
  383. )
  384. if restart_wait < DEFAULT_SESSION_RESTART_WAIT_SECONDS:
  385. raise DebugConfigurationError(
  386. "restart_wait must be at least "
  387. f"{DEFAULT_SESSION_RESTART_WAIT_SECONDS} seconds"
  388. )
  389. def _service_list_structure_for_log(plugin: ItaPlugin) -> str:
  390. raw = getattr(plugin, "last_service_list_structure", None)
  391. if not isinstance(raw, dict):
  392. return "not-available"
  393. allowed = {
  394. "path": raw.get("path"),
  395. "ready_state": raw.get("ready_state"),
  396. "table_count": int(raw.get("table_count") or 0),
  397. "row_count": int(raw.get("row_count") or 0),
  398. "booking_link_count": int(raw.get("booking_link_count") or 0),
  399. "target_link_count": int(raw.get("target_link_count") or 0),
  400. }
  401. return json.dumps(allowed, ensure_ascii=True, sort_keys=True)
  402. def run_soak(
  403. env: Mapping[str, str],
  404. *,
  405. apply: bool,
  406. non_interactive: bool,
  407. duration_seconds: int,
  408. min_wait: int,
  409. max_wait: int,
  410. max_session_attempts: int = DEFAULT_MAX_SESSION_ATTEMPTS,
  411. restart_wait: int = DEFAULT_SESSION_RESTART_WAIT_SECONDS,
  412. randint: Callable[[int, int], int] = random.randint,
  413. sleep: Callable[[float], None] = time.sleep,
  414. monotonic: Callable[[], float] = time.monotonic,
  415. wall_time: Callable[[], float] = time.time,
  416. plugin_factory: Callable[[str], ItaPlugin] | None = None,
  417. status_writer: Callable[[Mapping[str, object]], object] | None = None,
  418. prepared_calendar: CalendarSoakContext | None = None,
  419. restart_after_probes: int = 0,
  420. ) -> int:
  421. # The prepared instance owns its verified route. Do not reinterpret the
  422. # environment after ownership has transferred from the Calendar gate.
  423. route = prepared_calendar.route if prepared_calendar is not None else resolve_debug_route(env)
  424. route_log = (
  425. f"service_id={route.service_id} routing_key={route.routing_key}"
  426. )
  427. _validate_soak_limits(
  428. duration_seconds=duration_seconds,
  429. min_wait=min_wait,
  430. max_wait=max_wait,
  431. max_session_attempts=max_session_attempts,
  432. restart_wait=restart_wait,
  433. )
  434. if prepared_calendar is not None and min_wait < CALENDAR_MIN_WAIT_SECONDS:
  435. raise DebugConfigurationError("Calendar soak min_wait must be at least 300 seconds")
  436. if type(restart_after_probes) is not int or restart_after_probes < 0:
  437. raise DebugConfigurationError("restart_after_probes must be a nonnegative integer")
  438. if restart_after_probes and prepared_calendar is None:
  439. raise DebugConfigurationError("Controlled restart requires the prepared Calendar mode")
  440. if not apply:
  441. print(
  442. "ITA_SOAK_PREFLIGHT_OK "
  443. f"{route_log} "
  444. f"duration_seconds={duration_seconds} "
  445. f"min_wait={min_wait} max_wait={max_wait} "
  446. f"max_session_attempts={max_session_attempts} "
  447. "external_request=false"
  448. )
  449. return 0
  450. if prepared_calendar is None:
  451. chrome_path = resolve_chrome_path(env)
  452. os.environ["CHROME_BIN"] = str(chrome_path)
  453. username, password = _get_query_credentials(
  454. env,
  455. non_interactive=non_interactive,
  456. )
  457. profile_path = _prepare_query_profile(username)
  458. config = build_query_config(
  459. env,
  460. username=username,
  461. password=password,
  462. persistent_profile_path=profile_path,
  463. route=route,
  464. )
  465. else:
  466. config = prepared_calendar.config
  467. factory = plugin_factory or ItaPlugin
  468. writer = status_writer or write_soak_status
  469. apt_type = route.appointment_type()
  470. monitor_started = monotonic()
  471. started_wall = wall_time()
  472. plugin: ItaPlugin | None = prepared_calendar.initial_plugin if prepared_calendar else None
  473. current_session = 1 if prepared_calendar else 0
  474. probes = 0
  475. none_available = 0
  476. transient_errors = 0
  477. consecutive_transient_errors = 0
  478. session_attempts = current_session
  479. session_expirations = 0
  480. consecutive_session_expirations = 0
  481. session_reauthentication_times: list[float] = []
  482. browser_disconnects = 0
  483. consecutive_browser_disconnects = 0
  484. browser_connect_failures = 0
  485. consecutive_browser_connect_failures = 0
  486. exit_code = 0
  487. final_state = "running"
  488. final_reason = ""
  489. cleanup_error: Exception | None = None
  490. controlled_restart_done = False
  491. consecutive_calendar_network_errors = 0
  492. network_backoff = ExponentialBackoff(
  493. base_delay=restart_wait, max_delay=max_wait, jitter_range=(0, 0),
  494. )
  495. def elapsed_seconds() -> int:
  496. elapsed = max(0.0, monotonic() - monitor_started)
  497. return min(duration_seconds, int(elapsed))
  498. def status_body(state: str, *, code: int | None = None) -> dict:
  499. now = wall_time()
  500. current_monotonic = monotonic()
  501. reauthentications_last_hour = sum(
  502. 1
  503. for timestamp in session_reauthentication_times
  504. if 0
  505. <= current_monotonic - timestamp
  506. < SESSION_REAUTHENTICATION_WINDOW_SECONDS
  507. )
  508. body = {
  509. "state": state,
  510. "service_id": route.service_id,
  511. "routing_key": apt_type.routing_key,
  512. "duration_seconds": duration_seconds,
  513. "elapsed_seconds": elapsed_seconds(),
  514. "probes": probes,
  515. "none_available": none_available,
  516. "transient_errors": transient_errors,
  517. "consecutive_transient_errors": consecutive_transient_errors,
  518. "session_attempts": session_attempts,
  519. "session_expirations": session_expirations,
  520. "consecutive_session_expirations": (
  521. consecutive_session_expirations
  522. ),
  523. "session_reauthentications_last_hour": (
  524. reauthentications_last_hour
  525. ),
  526. "browser_disconnects": browser_disconnects,
  527. "consecutive_browser_disconnects": (
  528. consecutive_browser_disconnects
  529. ),
  530. "browser_connect_failures": browser_connect_failures,
  531. "consecutive_browser_connect_failures": (
  532. consecutive_browser_connect_failures
  533. ),
  534. "started_at": datetime.fromtimestamp(
  535. started_wall,
  536. tz=timezone.utc,
  537. ).isoformat(),
  538. "updated_at": datetime.fromtimestamp(
  539. now,
  540. tz=timezone.utc,
  541. ).isoformat(),
  542. }
  543. if code is not None:
  544. body["exit_code"] = code
  545. if final_reason:
  546. body["reason"] = final_reason
  547. if prepared_calendar is not None:
  548. body["calendar"] = prepared_calendar.safe_dict()
  549. return body
  550. def emit_status(state: str, *, code: int | None = None) -> None:
  551. writer(status_body(state, code=code))
  552. def cleanup_session(reason: str) -> None:
  553. nonlocal plugin, cleanup_error
  554. if plugin is None:
  555. return
  556. instance = plugin
  557. plugin = None
  558. try:
  559. try:
  560. if prepared_calendar is not None:
  561. prepared_calendar.close_session()
  562. finally:
  563. instance.cleanup()
  564. except Exception as exc:
  565. cleanup_error = exc
  566. raise
  567. print(
  568. "ITA_SOAK_SESSION_CLEANUP_OK "
  569. f"session={current_session} reason={reason}",
  570. flush=True,
  571. )
  572. def wait_before_session_restart() -> bool:
  573. remaining = duration_seconds - (monotonic() - monitor_started)
  574. wait_seconds = min(float(restart_wait), max(0.0, remaining))
  575. if wait_seconds:
  576. print(
  577. "ITA_SOAK_SESSION_RESTART_WAIT "
  578. f"seconds={int(wait_seconds)}",
  579. flush=True,
  580. )
  581. sleep(wait_seconds)
  582. return monotonic() - monitor_started < duration_seconds
  583. def start_session() -> bool:
  584. nonlocal plugin
  585. nonlocal current_session
  586. nonlocal session_attempts
  587. nonlocal transient_errors
  588. nonlocal browser_connect_failures
  589. nonlocal consecutive_browser_connect_failures
  590. while monotonic() - monitor_started < duration_seconds:
  591. if (
  592. max_session_attempts > 0
  593. and session_attempts >= max_session_attempts
  594. ):
  595. raise ItaSoakSessionAttemptLimitError
  596. session_attempts += 1
  597. current_session = session_attempts
  598. instance = factory("ita-soak-local-debug")
  599. plugin = instance
  600. instance.set_log(lambda message: print(message, flush=True))
  601. instance.set_config(config)
  602. try:
  603. instance.create_session()
  604. except BrowserConnectError:
  605. browser_connect_failures += 1
  606. transient_errors += 1
  607. consecutive_browser_connect_failures += 1
  608. print(
  609. "ITA_SOAK_BROWSER_CONNECT_FAILED "
  610. f"session_attempt={current_session} "
  611. f"consecutive={consecutive_browser_connect_failures} "
  612. f"total={browser_connect_failures}",
  613. flush=True,
  614. )
  615. cleanup_session("browser-connect-failed")
  616. if (
  617. consecutive_browser_connect_failures
  618. >= MAX_CONSECUTIVE_BROWSER_CONNECT_FAILURES
  619. ):
  620. raise ItaSoakBrowserConnectLimitError
  621. if not wait_before_session_restart():
  622. return False
  623. continue
  624. consecutive_browser_connect_failures = 0
  625. if prepared_calendar is not None:
  626. prepared_calendar.attach_restored_session(instance)
  627. print(
  628. "ITA_SOAK_SESSION_STARTED "
  629. f"session={current_session} "
  630. f"session_attempts={session_attempts}",
  631. flush=True,
  632. )
  633. return True
  634. return False
  635. try:
  636. if plugin is None:
  637. start_session()
  638. print(
  639. "ITA_SOAK_STARTED "
  640. f"{route_log} "
  641. f"duration_seconds={duration_seconds} "
  642. f"min_wait={min_wait} max_wait={max_wait}",
  643. flush=True,
  644. )
  645. emit_status("running")
  646. while True:
  647. if monotonic() - monitor_started >= duration_seconds:
  648. final_state = "completed"
  649. print(
  650. "ITA_SOAK_COMPLETED "
  651. f"duration_seconds={duration_seconds} probes={probes} "
  652. f"none_available={none_available} "
  653. f"transient_errors={transient_errors} "
  654. f"session_attempts={session_attempts}",
  655. flush=True,
  656. )
  657. break
  658. if (prepared_calendar is not None and restart_after_probes
  659. and not controlled_restart_done
  660. and prepared_calendar.positive_queries + prepared_calendar.negative_queries >= restart_after_probes):
  661. # A deliberate normal close tests ownership and restore, not
  662. # a fabricated website outage or an authentication expiry.
  663. controlled_restart_done = True
  664. emit_status("controlled_restart")
  665. cleanup_session("controlled-restart")
  666. if not wait_before_session_restart():
  667. continue
  668. start_session()
  669. emit_status("running")
  670. continue
  671. probes += 1
  672. print(
  673. "ITA_SOAK_PROBE_START "
  674. f"probe={probes} session={current_session} "
  675. f"elapsed_seconds={elapsed_seconds()}",
  676. flush=True,
  677. )
  678. try:
  679. if prepared_calendar is not None:
  680. if not plugin.health_check():
  681. if getattr(plugin, "_query_stop_reason", None) == "browser_disconnected":
  682. raise PageDisconnectedError("Calendar browser disconnected")
  683. raise SessionExpiredOrInvalidError("Calendar Session health expired")
  684. prepared_calendar.begin_query(plugin)
  685. result = plugin.query(apt_type)
  686. if prepared_calendar is not None:
  687. prepared_calendar.record_query(plugin, result)
  688. except ItaCalendarTransientError:
  689. if prepared_calendar is None:
  690. raise
  691. prepared_calendar.latest = None
  692. transient_errors += 1
  693. consecutive_calendar_network_errors += 1
  694. prepared_calendar.failure_reason = "transient-network"
  695. emit_status("cooldown")
  696. if consecutive_calendar_network_errors >= 2:
  697. final_state, final_reason, exit_code = "failed", "consecutive-calendar-network-errors", 1
  698. break
  699. remaining = duration_seconds - (monotonic() - monitor_started)
  700. sleep(min(network_backoff.calculate(consecutive_calendar_network_errors), max(0.0, remaining)))
  701. continue
  702. except ItaTargetServiceLinkMissingError:
  703. consecutive_session_expirations = 0
  704. consecutive_browser_disconnects = 0
  705. transient_errors += 1
  706. consecutive_transient_errors += 1
  707. print(
  708. "ITA_SERVICE_LIST_STRUCTURE="
  709. + _service_list_structure_for_log(plugin),
  710. flush=True,
  711. )
  712. print(
  713. "ITA_SOAK_TRANSIENT "
  714. "type=ItaTargetServiceLinkMissingError "
  715. f"consecutive={consecutive_transient_errors} "
  716. f"total={transient_errors}",
  717. flush=True,
  718. )
  719. if (
  720. consecutive_transient_errors
  721. >= MAX_CONSECUTIVE_TARGET_LINK_MISSES
  722. ):
  723. final_state = "failed"
  724. final_reason = "consecutive-target-link-misses"
  725. exit_code = 1
  726. print(
  727. "ITA_SOAK_FAILED "
  728. "reason=consecutive-target-link-misses",
  729. flush=True,
  730. )
  731. break
  732. except PageDisconnectedError:
  733. if prepared_calendar is not None:
  734. prepared_calendar.latest = None
  735. prepared_calendar.failure_reason = "browser-disconnected"
  736. browser_disconnects += 1
  737. transient_errors += 1
  738. consecutive_browser_disconnects += 1
  739. consecutive_transient_errors = 0
  740. print(
  741. "ITA_SOAK_BROWSER_DISCONNECTED "
  742. f"session={current_session} "
  743. f"consecutive={consecutive_browser_disconnects} "
  744. f"total={browser_disconnects}",
  745. flush=True,
  746. )
  747. cleanup_session("browser-disconnected")
  748. if (
  749. consecutive_browser_disconnects
  750. >= MAX_CONSECUTIVE_BROWSER_DISCONNECTS
  751. ):
  752. final_state = "failed"
  753. final_reason = "consecutive-browser-disconnects"
  754. exit_code = 1
  755. print(
  756. "ITA_SOAK_FAILED "
  757. "reason=consecutive-browser-disconnects",
  758. flush=True,
  759. )
  760. break
  761. if not wait_before_session_restart():
  762. continue
  763. start_session()
  764. emit_status("running")
  765. continue
  766. except SessionExpiredOrInvalidError:
  767. if prepared_calendar is not None:
  768. prepared_calendar.latest = None
  769. prepared_calendar.failure_reason = "session-expired"
  770. expiry_cause = (
  771. "configuredLifetime"
  772. if getattr(plugin, "_query_stop_reason", None) == "configured_lifetime_reached"
  773. else "authentication"
  774. )
  775. prepared_calendar.expiry_causes[expiry_cause] += 1
  776. session_expirations += 1
  777. consecutive_session_expirations += 1
  778. consecutive_transient_errors = 0
  779. consecutive_browser_disconnects = 0
  780. current_monotonic = monotonic()
  781. session_reauthentication_times[:] = [
  782. timestamp
  783. for timestamp in session_reauthentication_times
  784. if 0
  785. <= current_monotonic - timestamp
  786. < SESSION_REAUTHENTICATION_WINDOW_SECONDS
  787. ]
  788. session_reauthentication_times.append(current_monotonic)
  789. reauthentications_last_hour = len(
  790. session_reauthentication_times
  791. )
  792. print(
  793. "ITA_SOAK_SESSION_EXPIRED "
  794. f"session={current_session} "
  795. f"session_expirations={session_expirations} "
  796. f"consecutive={consecutive_session_expirations} "
  797. f"reauthentications_last_hour="
  798. f"{reauthentications_last_hour}",
  799. flush=True,
  800. )
  801. cleanup_session("session-expired")
  802. if (
  803. consecutive_session_expirations
  804. >= MAX_CONSECUTIVE_SESSION_EXPIRATIONS
  805. ):
  806. final_state = "failed"
  807. final_reason = "consecutive-session-expirations"
  808. exit_code = 1
  809. print(
  810. "ITA_SOAK_FAILED "
  811. "reason=consecutive-session-expirations",
  812. flush=True,
  813. )
  814. break
  815. if (
  816. reauthentications_last_hour
  817. > MAX_SESSION_REAUTHENTICATIONS_PER_WINDOW
  818. ):
  819. final_state = "failed"
  820. final_reason = "reauthentication-rate-limit"
  821. exit_code = 1
  822. print(
  823. "ITA_SOAK_FAILED "
  824. "reason=reauthentication-rate-limit",
  825. flush=True,
  826. )
  827. break
  828. if not wait_before_session_restart():
  829. continue
  830. start_session()
  831. emit_status("running")
  832. continue
  833. else:
  834. consecutive_calendar_network_errors = 0
  835. consecutive_transient_errors = 0
  836. consecutive_session_expirations = 0
  837. consecutive_browser_disconnects = 0
  838. if prepared_calendar is not None:
  839. if not result.success:
  840. none_available += 1
  841. elif _is_weak_entry_signal(result):
  842. _print_booking_form_structure(plugin, route=route)
  843. final_state = "weak_signal"
  844. final_reason = "booking-form"
  845. exit_code = 4
  846. print(
  847. "\aITA_SOAK_WEAK_SIGNAL "
  848. f"{route_log} probe={probes} "
  849. "evidence=booking-form",
  850. flush=True,
  851. )
  852. break
  853. elif _is_nonempty_query_result(result):
  854. final_state = "safe_stop"
  855. final_reason = "unexpected-non-empty-result"
  856. exit_code = 4
  857. print(
  858. "ITA_SOAK_SAFE_STOP "
  859. "reason=unexpected-non-empty-result",
  860. flush=True,
  861. )
  862. break
  863. if prepared_calendar is None:
  864. none_available += 1
  865. print(
  866. "ITA_SOAK_NONE_AVAILABLE "
  867. f"probe={probes} session={current_session}",
  868. flush=True,
  869. )
  870. emit_status("running")
  871. remaining = duration_seconds - (monotonic() - monitor_started)
  872. if remaining <= 0:
  873. continue
  874. wait_seconds = min(float(randint(min_wait, max_wait)), remaining)
  875. print(
  876. "ITA_SOAK_WAIT "
  877. f"next_probe={probes + 1} seconds={int(wait_seconds)}",
  878. flush=True,
  879. )
  880. sleep(wait_seconds)
  881. except ItaSoakBrowserConnectLimitError:
  882. final_state = "failed"
  883. final_reason = "consecutive-browser-connect-failures"
  884. exit_code = 1
  885. print(
  886. "ITA_SOAK_FAILED "
  887. "reason=consecutive-browser-connect-failures",
  888. flush=True,
  889. )
  890. except ItaSoakSessionAttemptLimitError:
  891. final_state = "failed"
  892. final_reason = "max-session-attempts-reached"
  893. exit_code = 1
  894. print(
  895. "ITA_SOAK_FAILED reason=max-session-attempts-reached",
  896. flush=True,
  897. )
  898. except ItaCalendarPreparationRequiredError:
  899. final_state, final_reason, exit_code = "needs_manual", "calendar-preparation-required", 4
  900. if prepared_calendar is not None:
  901. prepared_calendar.latest = None
  902. prepared_calendar.failure_reason = final_reason
  903. print("ITA_SOAK_NEEDS_MANUAL todo=ITA-OTP-01 otp_sent=false avanti_submitted=false", flush=True)
  904. except (PermissionDeniedError, ItaAuthenticationChallengeError):
  905. final_state, final_reason, exit_code = "needs_manual", "website-verification-required", 4
  906. print("ITA_SOAK_NEEDS_MANUAL reason=website-verification-required", flush=True)
  907. except RateLimiteddError:
  908. final_state, final_reason, exit_code = "rate_limited", "server-rate-limit", 5
  909. print("ITA_SOAK_RATE_LIMITED retry=false account_rotation=false", flush=True)
  910. except KeyboardInterrupt:
  911. final_state = "cancelled"
  912. final_reason = "operator-cancelled"
  913. exit_code = 130
  914. print(
  915. "\nITA_SOAK_CANCELLED reason=operator-cancelled",
  916. flush=True,
  917. )
  918. except (BizLogicError, CalendarGateError) as exc:
  919. final_state = "failed"
  920. final_reason = "business-error"
  921. exit_code = 1
  922. print(
  923. f"ITA_SOAK_FAILED type={type(exc).__name__} "
  924. f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(exc)}",
  925. flush=True,
  926. )
  927. if plugin is not None and prepared_calendar is None:
  928. _print_failure_context(plugin)
  929. except Exception as exc:
  930. final_state = "failed"
  931. final_reason = "unexpected-error"
  932. exit_code = 1
  933. print(
  934. f"ITA_SOAK_FAILED type={type(exc).__name__} "
  935. f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(exc)}",
  936. flush=True,
  937. )
  938. if plugin is not None and prepared_calendar is None:
  939. _print_failure_context(plugin)
  940. finally:
  941. if prepared_calendar is not None:
  942. if exit_code != 0:
  943. prepared_calendar.latest = None
  944. prepared_calendar.failure_reason = final_reason
  945. try:
  946. prepared_calendar.verify_end(plugin)
  947. except Exception as exc:
  948. print(f"ITA_SOAK_END_EVIDENCE_UNAVAILABLE type={type(exc).__name__}", flush=True)
  949. if exit_code == 0:
  950. has_queries = prepared_calendar.positive_queries + prepared_calendar.negative_queries > 0
  951. if (not has_queries or prepared_calendar.audit_gaps
  952. or not prepared_calendar.reservation_verified
  953. or prepared_calendar.queries_in_current_session == 0
  954. or consecutive_calendar_network_errors
  955. or consecutive_session_expirations or consecutive_browser_disconnects
  956. or (restart_after_probes and not controlled_restart_done)):
  957. final_state, final_reason, exit_code = "incomplete", "calendar-evidence-incomplete", 6
  958. try:
  959. cleanup_session("soak-finished")
  960. except Exception as exc:
  961. cleanup_error = exc
  962. if cleanup_error is not None:
  963. final_state = "failed"
  964. final_reason = "cleanup-failed"
  965. print(
  966. f"ITA_SOAK_CLEANUP_FAILED type={type(cleanup_error).__name__} "
  967. f"message={'details-withheld' if prepared_calendar else _popup_text_for_log(cleanup_error)}",
  968. file=sys.stderr,
  969. flush=True,
  970. )
  971. else:
  972. print("ITA_SOAK_CLEANUP_OK", flush=True)
  973. final_code = 3 if cleanup_error is not None else exit_code
  974. if prepared_calendar is not None and final_code != 0:
  975. prepared_calendar.failure_reason = final_reason
  976. prepared_calendar.latest = None
  977. try:
  978. emit_status(final_state, code=final_code)
  979. except Exception as exc:
  980. cleanup_error = cleanup_error or exc
  981. if prepared_calendar is not None:
  982. final_state, final_reason = "failed", "status-write-failed"
  983. prepared_calendar.failure_reason = final_reason
  984. print(
  985. f"ITA_SOAK_STATUS_FAILED type={type(exc).__name__}",
  986. file=sys.stderr,
  987. flush=True,
  988. )
  989. if prepared_calendar is not None:
  990. report = prepared_calendar.safe_dict()
  991. report.update({
  992. "state": final_state, "reason": final_reason,
  993. "elapsedSeconds": max(0, int(monotonic() - monitor_started)), "queryAttempts": probes,
  994. "sessionAttempts": session_attempts,
  995. "controlledRestart": controlled_restart_done,
  996. "exitCode": 3 if cleanup_error is not None else exit_code,
  997. "passed": cleanup_error is None and exit_code == 0,
  998. })
  999. print("ITA_SOAK_CALENDAR_RESULT=" + json.dumps(report, sort_keys=True), flush=True)
  1000. return 3 if cleanup_error is not None else exit_code
  1001. def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
  1002. parser = argparse.ArgumentParser(
  1003. description=(
  1004. "Duration-bounded route-configured Italy Query soak. Without --run, "
  1005. "performs a zero-network preflight only."
  1006. )
  1007. )
  1008. parser.add_argument(
  1009. "--calendar",
  1010. action="store_true",
  1011. help="Use one manual Calendar preparation, then the production Query and read-only recovery.",
  1012. )
  1013. parser.add_argument(
  1014. "--run",
  1015. action="store_true",
  1016. help="Start the real website soak.",
  1017. )
  1018. parser.add_argument(
  1019. "--non-interactive",
  1020. action="store_true",
  1021. help="Fail instead of prompting when local credentials are unavailable.",
  1022. )
  1023. parser.add_argument(
  1024. "--duration-seconds",
  1025. type=int,
  1026. default=None,
  1027. help=f"Soak duration in seconds (maximum: {DAY_SECONDS}).",
  1028. )
  1029. parser.add_argument(
  1030. "--min-wait",
  1031. type=int,
  1032. default=None,
  1033. help=f"Minimum seconds between probes (Calendar floor: {CALENDAR_MIN_WAIT_SECONDS}; entry-probe floor: {DEFAULT_MIN_WAIT_SECONDS}).",
  1034. )
  1035. parser.add_argument(
  1036. "--max-wait",
  1037. type=int,
  1038. default=None,
  1039. help="Maximum seconds between probes.",
  1040. )
  1041. parser.add_argument(
  1042. "--max-session-attempts",
  1043. type=int,
  1044. default=DEFAULT_MAX_SESSION_ATTEMPTS,
  1045. help=(
  1046. "Optional total Session-attempt limit; 0 uses the duration and "
  1047. "reauthentication circuit breakers only."
  1048. ),
  1049. )
  1050. parser.add_argument(
  1051. "--restart-wait",
  1052. type=int,
  1053. default=DEFAULT_SESSION_RESTART_WAIT_SECONDS,
  1054. help=(
  1055. "Minimum seconds before recreating an explicitly expired Session "
  1056. f"(floor: {DEFAULT_SESSION_RESTART_WAIT_SECONDS})."
  1057. ),
  1058. )
  1059. parser.add_argument(
  1060. "--restart-after-probes", type=int, default=0,
  1061. help="Calendar only: normally close/recreate once after this many successful queries (0 disables).",
  1062. )
  1063. return parser.parse_args(argv)
  1064. def main(argv: list[str] | None = None) -> int:
  1065. args = parse_args(argv)
  1066. try:
  1067. if args.calendar:
  1068. from tools.debug_ita_calendar_gate import build_calendar_gate_config, run_calendar_gate
  1069. env = dict(os.environ)
  1070. for key in (
  1071. "duration_seconds", "min_wait", "max_wait", "max_session_attempts",
  1072. "restart_wait", "restart_after_probes",
  1073. ):
  1074. value = getattr(args, key)
  1075. if value is not None:
  1076. env[f"ITA_SOAK_{key.upper()}"] = str(value)
  1077. limits = calendar_soak_limits(env)
  1078. # Configuration only: no credential lookup, profile creation, or login.
  1079. build_calendar_gate_config(
  1080. env, username="preflight@example.invalid", password="unused-preflight",
  1081. )
  1082. if not args.run:
  1083. print("ITA_SOAK_CALENDAR_PREFLIGHT_OK " + json.dumps(limits, sort_keys=True)
  1084. + " external_request=false", flush=True)
  1085. return 0
  1086. if args.non_interactive:
  1087. raise DebugConfigurationError("Calendar preparation currently requires manual OTP input")
  1088. env["ITA_QCAL_MODE"] = "calendar_soak"
  1089. return run_calendar_gate(env)
  1090. if args.restart_after_probes:
  1091. raise DebugConfigurationError("--restart-after-probes requires --calendar")
  1092. return run_soak(
  1093. os.environ,
  1094. apply=args.run,
  1095. non_interactive=args.non_interactive,
  1096. duration_seconds=args.duration_seconds if args.duration_seconds is not None else DAY_SECONDS,
  1097. min_wait=args.min_wait if args.min_wait is not None else DEFAULT_MIN_WAIT_SECONDS,
  1098. max_wait=args.max_wait if args.max_wait is not None else DEFAULT_MAX_WAIT_SECONDS,
  1099. max_session_attempts=args.max_session_attempts,
  1100. restart_wait=args.restart_wait,
  1101. )
  1102. except (DebugConfigurationError, CalendarGateError) as exc:
  1103. print(f"ITA_SOAK_CONFIG_ERROR: {exc}", file=sys.stderr)
  1104. return 2
  1105. if __name__ == "__main__":
  1106. raise SystemExit(main())