test_ita_calendar_soak.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. """Exercise the existing soak loop with real Italy Query and fake browser IO."""
  2. import io
  3. import json
  4. import tempfile
  5. import time
  6. import unittest
  7. from contextlib import ExitStack, redirect_stderr, redirect_stdout
  8. from dataclasses import fields, replace
  9. from pathlib import Path
  10. from unittest.mock import Mock, patch
  11. from DrissionPage.errors import BrowserConnectError
  12. from plugins.ita_plugin import ItaPlugin
  13. from test.ita.test_debug_ita_production_query_gate import gate_env
  14. from test.ita.test_debug_ita_soak import FakeClock
  15. from test.ita.test_ita_query_recovery import RecoveryPage, prepared_plugin
  16. from tools.debug_ita_calendar_gate import CalendarRequestAudit, ReservationSnapshot
  17. from tools.debug_ita_soak import (
  18. CalendarSoakContext, calendar_soak_limits, main, run_soak,
  19. )
  20. from tools.debug_ita_session import DebugConfigurationError
  21. class CalendarSoakLifecycleTests(unittest.TestCase):
  22. def run_case(
  23. self, outcomes=("available",), *, restored_outcomes=("available",),
  24. restore_outcome="calendar", duration=900, restart=0,
  25. max_sessions=0, end_snapshot=None, cleanup_failure=False,
  26. create_outcomes=("ok",), interrupt_wait=False, interrupt_health=False,
  27. soak_env=None, checkpoint_mutator=None,
  28. ):
  29. clock = FakeClock()
  30. output = io.StringIO()
  31. statuses = []
  32. instances = []
  33. pages = []
  34. with tempfile.TemporaryDirectory() as task_tmp, ExitStack() as stack:
  35. profile = Path(task_tmp) / "profile"
  36. profile.mkdir(mode=0o700)
  37. profile = profile.resolve() # Match set_config's canonical macOS /private/var path.
  38. stack.enter_context(patch.object(
  39. CalendarRequestAudit, "settled_snapshot", CalendarRequestAudit.snapshot,
  40. ))
  41. stack.enter_context(patch("plugins.ita_plugin.time.sleep", clock.sleep))
  42. stack.enter_context(patch(
  43. "tools.debug_ita_calendar_gate.read_reservation_snapshot",
  44. return_value=end_snapshot or ReservationSnapshot(0, 0),
  45. ))
  46. credentials = stack.enter_context(patch(
  47. "tools.debug_ita_soak._get_query_credentials",
  48. side_effect=AssertionError("must not reload credentials after preparation"),
  49. ))
  50. def make_instance(query_outcomes, *, restoring=False):
  51. page = RecoveryPage(query_outcomes, restore_outcome=restore_outcome)
  52. plugin, route = prepared_plugin(page)
  53. plugin.instance_id = f"offline-{len(instances)}"
  54. plugin._transient_user_data_path = str(Path(task_tmp) / plugin.instance_id / "user-data")
  55. plugin.root_workspace = str(Path(task_tmp) / plugin.instance_id)
  56. plugin.user_data_path = str(profile)
  57. plugin.free_config["persistent_browser_profile_path"] = str(profile)
  58. plugin.tunnel = None
  59. plugin._record_calendar_prepared_route(plugin._resolve_route(route.appointment_type()), role="query")
  60. plugin.book = Mock(side_effect=AssertionError("Book must not run in Query soak"))
  61. create_index = max(0, len(instances) - 1)
  62. create_outcome = create_outcomes[min(create_index, len(create_outcomes) - 1)]
  63. def create():
  64. plugin._reset_calendar_state()
  65. if create_outcome == "connect-error":
  66. raise BrowserConnectError("offline connection failed")
  67. plugin.page.url = "https://prenotami.esteri.it/UserArea"
  68. plugin.is_healthy = True
  69. plugin.session_create_time = time.time()
  70. plugin.create_session = Mock(side_effect=create)
  71. def cleanup():
  72. page.driver.is_running = False
  73. plugin.page = None
  74. plugin.is_healthy = False
  75. plugin._reset_calendar_state()
  76. if cleanup_failure:
  77. raise RuntimeError("private@example.test secret-token")
  78. plugin.cleanup = Mock(side_effect=cleanup)
  79. instances.append(plugin)
  80. pages.append(page)
  81. return plugin, route
  82. initial, route = make_instance(outcomes)
  83. audit = CalendarRequestAudit(initial.page, include_time_slots=True)
  84. audit.start()
  85. initial.page.emit("/BookingCalendar/GenerateOTP?ServiceID=9001", "GET")
  86. initial.page.emit("/Services/Booking/9001", "POST")
  87. context = CalendarSoakContext(
  88. plugin=initial, route=route, request_audit=audit, before=ReservationSnapshot(0, 0),
  89. )
  90. if checkpoint_mutator is not None:
  91. checkpoint_mutator(context, initial, route)
  92. if interrupt_health:
  93. initial.page.run_js = Mock(side_effect=KeyboardInterrupt)
  94. def factory(_group_id):
  95. return make_instance(restored_outcomes, restoring=True)[0]
  96. def sleep(seconds):
  97. if interrupt_wait:
  98. raise KeyboardInterrupt
  99. clock.sleep(seconds)
  100. with redirect_stdout(output), redirect_stderr(output):
  101. code = run_soak(
  102. gate_env() if soak_env is None else soak_env, apply=True, non_interactive=True,
  103. duration_seconds=duration, min_wait=300, max_wait=300,
  104. max_session_attempts=max_sessions, restart_after_probes=restart,
  105. plugin_factory=factory, prepared_calendar=context,
  106. sleep=sleep, monotonic=clock.monotonic, wall_time=clock.wall_time,
  107. randint=lambda minimum, maximum: minimum,
  108. status_writer=lambda body: statuses.append(json.loads(json.dumps(body))),
  109. )
  110. credentials.assert_not_called()
  111. for instance in instances:
  112. instance.book.assert_not_called()
  113. for page in pages:
  114. self.assertEqual(page.driver.immediate_event_handlers, {})
  115. result_lines = [line for line in output.getvalue().splitlines()
  116. if line.startswith("ITA_SOAK_CALENDAR_RESULT=")]
  117. self.assertEqual(len(result_lines), 1)
  118. report = json.loads(result_lines[0].split("=", 1)[1])
  119. # Assert ownership before object finalizers can call cleanup again.
  120. cleanup_calls = [instance.cleanup.call_count for instance in instances]
  121. for instance in instances:
  122. instance.cleanup = Mock() # neutralize __del__, not loop verification
  123. return SimpleResult(code, report, statuses, output.getvalue(), instances, pages, cleanup_calls)
  124. def test_positive_negative_positive_continues_with_fresh_results(self):
  125. result = self.run_case(["available", "none", "available"])
  126. self.assertEqual(result.code, 0, result.output)
  127. self.assertTrue(result.report["passed"])
  128. self.assertEqual(result.report["positiveQueries"], 2)
  129. self.assertEqual(result.report["negativeQueries"], 1)
  130. self.assertEqual(result.report["queryRequests"]["availabilityRequests"], 5)
  131. self.assertEqual(result.report["queryRequests"]["timeSlotRequests"], 3)
  132. self.assertEqual(result.cleanup_calls, [1])
  133. self.assertEqual(len(result.instances), 1)
  134. self.assertEqual(result.report["latestQuery"]["earliestDate"], "2026-08-12")
  135. self.assertNotIn("private@example.test", result.output)
  136. def test_controlled_restart_uses_same_profile_and_new_query_without_otp(self):
  137. result = self.run_case(restart=1)
  138. self.assertEqual(result.code, 0, result.output)
  139. self.assertTrue(result.report["controlledRestart"])
  140. self.assertEqual(result.report["restores"], 1)
  141. self.assertEqual(result.cleanup_calls, [1, 1])
  142. self.assertEqual(result.pages[0].refresh_calls, 1)
  143. self.assertGreater(result.pages[1].refresh_calls, 0)
  144. self.assertEqual(result.pages[1].get_calls, ["https://prenotami.esteri.it/BookingCalendar"])
  145. for key in ("otpRequests", "avantiPosts", "finalRequests"):
  146. self.assertEqual(result.report["queryRequests"][key], 0)
  147. def test_soak_checkpoint_is_local_and_contains_no_credentials_or_inventory(self):
  148. def inspect(context, plugin, _route):
  149. self.assertEqual(
  150. {item.name for item in fields(context.checkpoint)},
  151. {"route", "username", "profile_path", "calendar_url", "runtime_service"},
  152. )
  153. body = repr(context.checkpoint)
  154. self.assertNotIn(plugin.config.account.username, body)
  155. self.assertNotIn(plugin.config.account.password, body)
  156. self.assertNotIn("earliest", body.lower())
  157. self.assertNotIn("slot", body.lower())
  158. result = self.run_case(restart=1, checkpoint_mutator=inspect)
  159. self.assertEqual(result.code, 0, result.output)
  160. self.assertEqual(result.report["restores"], 1)
  161. def test_soak_restore_rejects_changed_owner_profile_or_route_before_navigation(self):
  162. def mutate(field_name):
  163. def apply(context, _plugin, _route):
  164. checkpoint = context.checkpoint
  165. if field_name == "username":
  166. context.checkpoint = replace(checkpoint, username="other@example.test")
  167. elif field_name == "profile":
  168. context.checkpoint = replace(checkpoint, profile_path="/offline/other-profile")
  169. else:
  170. context.checkpoint = replace(
  171. checkpoint, route=replace(checkpoint.route, service_id=9002),
  172. )
  173. return apply
  174. for field_name in ("username", "profile", "route"):
  175. with self.subTest(field=field_name):
  176. result = self.run_case(restart=1, checkpoint_mutator=mutate(field_name))
  177. self.assertNotEqual(result.code, 0, result.output)
  178. self.assertEqual(result.pages[1].get_calls, [])
  179. self.assertEqual(result.report["restores"], 0)
  180. def test_soak_restore_rejects_foreign_or_unobserved_calendar_url_before_navigation(self):
  181. for calendar_url in (
  182. "https://foreign.example/BookingCalendar",
  183. "https://prenotami.esteri.it/BookingCalendar?unknown=1",
  184. ):
  185. def mutate(context, _plugin, _route, value=calendar_url):
  186. context.checkpoint = replace(context.checkpoint, calendar_url=value)
  187. with self.subTest(calendar_url=calendar_url):
  188. result = self.run_case(restart=1, checkpoint_mutator=mutate)
  189. self.assertNotEqual(result.code, 0, result.output)
  190. self.assertEqual(result.pages[1].get_calls, [])
  191. self.assertEqual(result.report["restores"], 0)
  192. def test_prepared_context_owns_route_without_reloading_environment(self):
  193. result = self.run_case(restart=1, soak_env={})
  194. self.assertEqual(result.code, 0, result.output)
  195. self.assertEqual(result.report["restores"], 1)
  196. self.assertEqual(result.cleanup_calls, [1, 1])
  197. self.assertEqual(result.statuses[-1]["service_id"], 9001)
  198. def test_expired_session_rebuilds_then_continues_new_queries(self):
  199. result = self.run_case(["expired"])
  200. self.assertEqual(result.code, 0, result.output)
  201. self.assertEqual(result.report["restores"], 1)
  202. self.assertEqual(result.cleanup_calls, [1, 1])
  203. self.assertGreater(result.report["positiveQueries"], 0)
  204. self.assertEqual(result.instances[1].create_session.call_count, 1)
  205. def test_disconnect_recovers_but_reports_audit_gap_not_false_pass(self):
  206. result = self.run_case(["disconnect"])
  207. self.assertEqual(result.code, 6, result.output)
  208. self.assertFalse(result.report["passed"])
  209. self.assertEqual(result.report["auditGaps"], 1)
  210. self.assertEqual(result.report["restores"], 1)
  211. self.assertGreater(result.report["positiveQueries"], 0)
  212. self.assertEqual(result.cleanup_calls, [1, 1])
  213. def test_two_expirations_stop_without_third_session(self):
  214. result = self.run_case(["expired"], restored_outcomes=["expired"])
  215. self.assertEqual(result.code, 1, result.output)
  216. self.assertEqual(result.report["reason"], "consecutive-session-expirations")
  217. self.assertEqual(len(result.instances), 2)
  218. self.assertEqual(result.cleanup_calls, [1, 1])
  219. def test_transient_network_failure_retries_same_browser(self):
  220. result = self.run_case(["timeout", "available"])
  221. self.assertEqual(result.code, 0, result.output)
  222. self.assertEqual(len(result.instances), 1)
  223. self.assertGreater(result.report["positiveQueries"], 0)
  224. self.assertEqual(result.report["restores"], 0)
  225. def test_two_transient_failures_stop_without_relogin_or_third_probe(self):
  226. result = self.run_case(["timeout"])
  227. self.assertEqual(result.code, 1, result.output)
  228. self.assertEqual(result.pages[0].attempts, 2)
  229. self.assertEqual(result.report["reason"], "consecutive-calendar-network-errors")
  230. self.assertEqual(len(result.instances), 1)
  231. def test_otp_reentry_is_pending_todo_not_attempted_again(self):
  232. result = self.run_case(["expired"], restore_outcome="form")
  233. self.assertEqual(result.code, 4, result.output)
  234. self.assertEqual(result.report["state"], "needs_manual")
  235. self.assertEqual(result.report["automaticOtpTodo"], "ITA-OTP-01")
  236. self.assertIsNone(result.report["latestQuery"])
  237. self.assertEqual(result.pages[1].refresh_calls, 0)
  238. self.assertEqual(result.report["queryRequests"]["otpRequests"], 0)
  239. self.assertEqual(result.report["queryRequests"]["avantiPosts"], 0)
  240. def test_rate_limit_stops_without_account_rotation_or_relogin(self):
  241. result = self.run_case(["available", "rate"])
  242. self.assertEqual(result.code, 5, result.output)
  243. self.assertEqual(result.report["state"], "rate_limited")
  244. self.assertEqual(len(result.instances), 1)
  245. self.assertEqual(result.pages[0].attempts, 2)
  246. self.assertIsNone(result.report["latestQuery"])
  247. def test_permission_denied_stops_for_operator(self):
  248. result = self.run_case(["denied"])
  249. self.assertEqual(result.code, 4, result.output)
  250. self.assertEqual(result.report["state"], "needs_manual")
  251. self.assertEqual(result.pages[0].attempts, 1)
  252. def test_unknown_schema_stops_without_false_inventory_or_secret_log(self):
  253. result = self.run_case(["malformed"])
  254. self.assertEqual(result.code, 1, result.output)
  255. self.assertEqual(result.report["positiveQueries"], 0)
  256. self.assertEqual(result.report["negativeQueries"], 0)
  257. self.assertEqual(result.pages[0].attempts, 1)
  258. self.assertNotIn("secret-token", result.output)
  259. self.assertNotIn("private@example.test", result.output)
  260. def test_unexpected_write_request_cannot_pass(self):
  261. result = self.run_case(["unexpected-otp"])
  262. self.assertNotEqual(result.code, 0, result.output)
  263. self.assertFalse(result.report["passed"])
  264. self.assertEqual(result.report["queryRequests"]["otpRequests"], 1)
  265. self.assertEqual(result.report["positiveQueries"], 0)
  266. def test_wrong_service_after_relogin_stops_without_querying_it(self):
  267. result = self.run_case(["expired"], restore_outcome="wrong-service")
  268. self.assertEqual(result.code, 1, result.output)
  269. self.assertEqual(result.pages[1].refresh_calls, 0)
  270. self.assertFalse(result.report["passed"])
  271. def test_unverified_reservation_counts_cannot_pass(self):
  272. result = self.run_case(end_snapshot=ReservationSnapshot(1, 0))
  273. self.assertEqual(result.code, 6, result.output)
  274. self.assertFalse(result.report["passed"])
  275. self.assertFalse(result.report["reservationVerified"])
  276. def test_cancel_does_not_recreate_or_submit(self):
  277. result = self.run_case(interrupt_wait=True)
  278. self.assertEqual(result.code, 130, result.output)
  279. self.assertEqual(result.cleanup_calls, [1])
  280. self.assertFalse(result.report["passed"])
  281. def test_cancel_during_health_probe_cleans_up_without_relogin(self):
  282. result = self.run_case(interrupt_health=True)
  283. self.assertEqual(result.code, 130, result.output)
  284. self.assertEqual(result.cleanup_calls, [1])
  285. self.assertEqual(len(result.instances), 1)
  286. self.assertEqual(result.pages[0].attempts, 0)
  287. self.assertFalse(result.report["passed"])
  288. def test_cleanup_failure_is_not_reported_as_success(self):
  289. result = self.run_case(cleanup_failure=True)
  290. self.assertEqual(result.code, 3, result.output)
  291. self.assertFalse(result.report["passed"])
  292. self.assertNotIn("secret-token", result.output)
  293. def test_connect_error_reuses_existing_bounded_creation_retry(self):
  294. result = self.run_case(["expired"], create_outcomes=["connect-error", "ok"])
  295. self.assertEqual(result.code, 0, result.output)
  296. self.assertEqual(len(result.instances), 3)
  297. self.assertEqual(result.cleanup_calls, [1, 1, 1])
  298. def test_session_attempt_cap_stops_instead_of_silently_finishing(self):
  299. result = self.run_case(["expired"], max_sessions=1)
  300. self.assertEqual(result.code, 1, result.output)
  301. self.assertEqual(result.report["reason"], "max-session-attempts-reached")
  302. self.assertEqual(len(result.instances), 1)
  303. def test_failed_cleanup_during_recovery_prevents_another_browser(self):
  304. result = self.run_case(["expired"], cleanup_failure=True)
  305. self.assertEqual(result.code, 3, result.output)
  306. self.assertEqual(result.report["reason"], "cleanup-failed")
  307. self.assertEqual(len(result.instances), 1)
  308. self.assertNotIn("ITA_SOAK_CLEANUP_OK", result.output)
  309. def test_two_connection_failures_stop_using_existing_cap(self):
  310. result = self.run_case(["expired"], create_outcomes=["connect-error"])
  311. self.assertEqual(result.code, 1, result.output)
  312. self.assertEqual(result.report["reason"], "consecutive-browser-connect-failures")
  313. self.assertEqual(len(result.instances), 3) # prepared + two failed starts
  314. def test_two_disconnections_stop_without_third_browser(self):
  315. result = self.run_case(["disconnect"], restored_outcomes=["disconnect"])
  316. self.assertEqual(result.code, 1, result.output)
  317. self.assertEqual(result.report["reason"], "consecutive-browser-disconnects")
  318. self.assertEqual(len(result.instances), 2)
  319. def test_hourly_reauthentication_cap_survives_successful_queries_between_failures(self):
  320. result = self.run_case(["expired"], restored_outcomes=["available", "expired"], duration=7200)
  321. self.assertEqual(result.code, 1, result.output)
  322. self.assertEqual(result.report["reason"], "reauthentication-rate-limit")
  323. self.assertEqual(len(result.instances), 7)
  324. def test_deadline_before_successful_retry_is_incomplete_not_a_pass(self):
  325. result = self.run_case(["timeout"], duration=60)
  326. self.assertEqual(result.code, 6, result.output)
  327. self.assertEqual(result.report["positiveQueries"], 0)
  328. def test_unexercised_requested_restart_is_not_marked_verified(self):
  329. result = self.run_case(restart=100)
  330. self.assertEqual(result.code, 6, result.output)
  331. self.assertFalse(result.report["controlledRestart"])
  332. class SimpleResult:
  333. def __init__(self, code, report, statuses, output, instances, pages, cleanup_calls):
  334. self.code, self.report, self.statuses, self.output = code, report, statuses, output
  335. self.instances, self.pages, self.cleanup_calls = instances, pages, cleanup_calls
  336. class CalendarSoakCliTests(unittest.TestCase):
  337. def test_gate_transfers_the_same_prepared_instance_and_cleanup_once(self):
  338. from test.ita import test_debug_ita_production_query_gate as workflow
  339. from tools.debug_ita_calendar_gate import CalendarGateMode
  340. def soak(_env, *, plugin, route, request_audit, before):
  341. self.assertEqual(route.service_id, 9001)
  342. self.assertEqual(before, ReservationSnapshot(0, 0))
  343. self.assertEqual(request_audit.snapshot()["otpRequests"], 1)
  344. self.assertEqual(request_audit.snapshot()["avantiPosts"], 1)
  345. plugin.page.set.auto_handle_alert.assert_called_with(accept=False)
  346. request_audit.stop()
  347. plugin.cleanup()
  348. return 0
  349. with patch("tools.debug_ita_calendar_gate.resolve_qcal_mode", return_value=CalendarGateMode.CALENDAR_SOAK), \
  350. patch("tools.debug_ita_soak.run_prepared_calendar_soak", side_effect=soak) as handoff:
  351. code, output, plugin, page, mocks, otp_reader = workflow.ProductionGateWorkflowTests()._run_gate()
  352. self.assertEqual(code, 0, output)
  353. handoff.assert_called_once()
  354. otp_reader.assert_called_once()
  355. plugin.cleanup.assert_called_once()
  356. mocks["run_production_query_acceptance"].assert_not_called()
  357. mocks["scan_calendar_horizon"].assert_not_called()
  358. def test_preflight_checks_calendar_config_without_credentials_or_browser(self):
  359. output = io.StringIO()
  360. with patch.dict("os.environ", gate_env(), clear=True), \
  361. patch("tools.debug_ita_calendar_gate.run_calendar_gate") as run, \
  362. patch("tools.debug_ita_soak._get_query_credentials") as credentials, \
  363. redirect_stdout(output):
  364. code = main(["--calendar", "--duration-seconds", "900"])
  365. self.assertEqual(code, 0)
  366. run.assert_not_called()
  367. credentials.assert_not_called()
  368. self.assertIn("external_request=false", output.getvalue())
  369. def test_unsafe_cadence_is_rejected_before_gate_or_otp(self):
  370. with patch.dict("os.environ", gate_env(), clear=True), \
  371. patch("tools.debug_ita_calendar_gate.run_calendar_gate") as run, \
  372. redirect_stderr(io.StringIO()):
  373. code = main(["--calendar", "--run", "--min-wait", "10"])
  374. self.assertEqual(code, 2)
  375. run.assert_not_called()
  376. def test_actual_gate_is_only_entered_with_run(self):
  377. with patch.dict("os.environ", gate_env(), clear=True), \
  378. patch("tools.debug_ita_calendar_gate.run_calendar_gate", return_value=0) as run:
  379. self.assertEqual(main(["--calendar", "--run", "--restart-after-probes", "1"]), 0)
  380. env = run.call_args.args[0]
  381. self.assertEqual(env["ITA_QCAL_MODE"], "calendar_soak")
  382. self.assertEqual(env["ITA_SOAK_RESTART_AFTER_PROBES"], "1")
  383. def test_limits_reject_bad_values_before_external_actions(self):
  384. for value in ("nan", "0", "86401"):
  385. with self.subTest(value=value), self.assertRaises(DebugConfigurationError):
  386. calendar_soak_limits({"ITA_SOAK_DURATION_SECONDS": value})
  387. if __name__ == "__main__":
  388. unittest.main()