test_ita_calendar_soak.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. """Exercise the existing soak loop with real Italy Query and fake browser IO."""
  2. import io
  3. import json
  4. import tempfile
  5. import time
  6. import unittest
  7. from contextlib import ExitStack, redirect_stderr, redirect_stdout
  8. from pathlib import Path
  9. from unittest.mock import Mock, patch
  10. from DrissionPage.errors import BrowserConnectError
  11. from plugins.ita_plugin import ItaPlugin
  12. from test.test_debug_ita_production_query_gate import gate_env
  13. from test.test_debug_ita_soak import FakeClock
  14. from test.test_ita_query_recovery import RecoveryPage, prepared_plugin
  15. from tools.debug_ita_calendar_gate import CalendarRequestAudit, ReservationSnapshot
  16. from tools.debug_ita_soak import (
  17. CalendarSoakContext, calendar_soak_limits, main, run_soak,
  18. )
  19. from tools.debug_ita_session import DebugConfigurationError
  20. class CalendarSoakLifecycleTests(unittest.TestCase):
  21. def run_case(
  22. self, outcomes=("available",), *, restored_outcomes=("available",),
  23. restore_outcome="calendar", duration=900, restart=0,
  24. max_sessions=0, end_snapshot=None, cleanup_failure=False,
  25. create_outcomes=("ok",), interrupt_wait=False, interrupt_health=False,
  26. soak_env=None,
  27. ):
  28. clock = FakeClock()
  29. output = io.StringIO()
  30. statuses = []
  31. instances = []
  32. pages = []
  33. with tempfile.TemporaryDirectory() as task_tmp, ExitStack() as stack:
  34. profile = Path(task_tmp) / "profile"
  35. profile.mkdir(mode=0o700)
  36. profile = profile.resolve() # Match set_config's canonical macOS /private/var path.
  37. stack.enter_context(patch.object(
  38. CalendarRequestAudit, "settled_snapshot", CalendarRequestAudit.snapshot,
  39. ))
  40. stack.enter_context(patch("plugins.ita_calendar_query.time.sleep", clock.sleep))
  41. stack.enter_context(patch(
  42. "tools.debug_ita_calendar_gate.read_reservation_snapshot",
  43. return_value=end_snapshot or ReservationSnapshot(0, 0),
  44. ))
  45. credentials = stack.enter_context(patch(
  46. "tools.debug_ita_soak._get_query_credentials",
  47. side_effect=AssertionError("must not reload credentials after preparation"),
  48. ))
  49. def make_instance(query_outcomes, *, restoring=False):
  50. page = RecoveryPage(query_outcomes, restore_outcome=restore_outcome)
  51. plugin, route = prepared_plugin(page)
  52. plugin.instance_id = f"offline-{len(instances)}"
  53. plugin._transient_user_data_path = str(Path(task_tmp) / plugin.instance_id / "user-data")
  54. plugin.root_workspace = str(Path(task_tmp) / plugin.instance_id)
  55. plugin._persistent_profile_path = profile
  56. plugin.user_data_path = str(profile)
  57. plugin.free_config["persistent_browser_profile_path"] = str(profile)
  58. plugin._profile_lock_handle = None
  59. plugin.tunnel = None
  60. plugin.book = Mock(side_effect=AssertionError("Book must not run in Query soak"))
  61. create_index = max(0, len(instances) - 1)
  62. create_outcome = create_outcomes[min(create_index, len(create_outcomes) - 1)]
  63. def create():
  64. plugin._clear_calendar_query_state()
  65. if create_outcome == "connect-error":
  66. raise BrowserConnectError("offline connection failed")
  67. plugin.page.url = "https://prenotami.esteri.it/UserArea"
  68. plugin.is_healthy = True
  69. plugin.session_create_time = time.time()
  70. plugin.create_session = Mock(side_effect=create)
  71. def cleanup():
  72. page.driver.is_running = False
  73. plugin.page = None
  74. plugin.is_healthy = False
  75. plugin._clear_calendar_query_state()
  76. if cleanup_failure:
  77. raise RuntimeError("private@example.test secret-token")
  78. plugin.cleanup = Mock(side_effect=cleanup)
  79. instances.append(plugin)
  80. pages.append(page)
  81. return plugin, route
  82. initial, route = make_instance(outcomes)
  83. audit = CalendarRequestAudit(initial.page, include_time_slots=True)
  84. audit.start()
  85. initial.page.emit("/BookingCalendar/GenerateOTP?ServiceID=9001", "GET")
  86. initial.page.emit("/Services/Booking/9001", "POST")
  87. context = CalendarSoakContext(
  88. plugin=initial, route=route, request_audit=audit, before=ReservationSnapshot(0, 0),
  89. )
  90. if interrupt_health:
  91. initial.page.run_js = Mock(side_effect=KeyboardInterrupt)
  92. def factory(_group_id):
  93. return make_instance(restored_outcomes, restoring=True)[0]
  94. def sleep(seconds):
  95. if interrupt_wait:
  96. raise KeyboardInterrupt
  97. clock.sleep(seconds)
  98. with redirect_stdout(output), redirect_stderr(output):
  99. code = run_soak(
  100. gate_env() if soak_env is None else soak_env, apply=True, non_interactive=True,
  101. duration_seconds=duration, min_wait=300, max_wait=300,
  102. max_session_attempts=max_sessions, restart_after_probes=restart,
  103. plugin_factory=factory, prepared_calendar=context,
  104. sleep=sleep, monotonic=clock.monotonic, wall_time=clock.wall_time,
  105. randint=lambda minimum, maximum: minimum,
  106. status_writer=lambda body: statuses.append(json.loads(json.dumps(body))),
  107. )
  108. credentials.assert_not_called()
  109. for instance in instances:
  110. instance.book.assert_not_called()
  111. for page in pages:
  112. self.assertEqual(page.driver.immediate_event_handlers, {})
  113. result_lines = [line for line in output.getvalue().splitlines()
  114. if line.startswith("ITA_SOAK_CALENDAR_RESULT=")]
  115. self.assertEqual(len(result_lines), 1)
  116. report = json.loads(result_lines[0].split("=", 1)[1])
  117. # Assert ownership before object finalizers can call cleanup again.
  118. cleanup_calls = [instance.cleanup.call_count for instance in instances]
  119. for instance in instances:
  120. instance.cleanup = Mock() # neutralize __del__, not loop verification
  121. return SimpleResult(code, report, statuses, output.getvalue(), instances, pages, cleanup_calls)
  122. def test_positive_negative_positive_continues_with_fresh_results(self):
  123. result = self.run_case(["available", "none", "available"])
  124. self.assertEqual(result.code, 0, result.output)
  125. self.assertTrue(result.report["passed"])
  126. self.assertEqual(result.report["positiveQueries"], 2)
  127. self.assertEqual(result.report["negativeQueries"], 1)
  128. self.assertEqual(result.report["queryRequests"]["availabilityRequests"], 5)
  129. self.assertEqual(result.report["queryRequests"]["timeSlotRequests"], 3)
  130. self.assertEqual(result.cleanup_calls, [1])
  131. self.assertEqual(len(result.instances), 1)
  132. self.assertEqual(result.report["latestQuery"]["earliestDate"], "2026-08-12")
  133. self.assertNotIn("private@example.test", result.output)
  134. def test_controlled_restart_uses_same_profile_and_new_query_without_otp(self):
  135. result = self.run_case(restart=1)
  136. self.assertEqual(result.code, 0, result.output)
  137. self.assertTrue(result.report["controlledRestart"])
  138. self.assertEqual(result.report["restores"], 1)
  139. self.assertEqual(result.cleanup_calls, [1, 1])
  140. self.assertEqual(result.pages[0].refresh_calls, 1)
  141. self.assertGreater(result.pages[1].refresh_calls, 0)
  142. self.assertEqual(result.pages[1].get_calls, ["https://prenotami.esteri.it/BookingCalendar"])
  143. for key in ("otpRequests", "avantiPosts", "finalRequests"):
  144. self.assertEqual(result.report["queryRequests"][key], 0)
  145. def test_prepared_context_owns_route_without_reloading_environment(self):
  146. result = self.run_case(restart=1, soak_env={})
  147. self.assertEqual(result.code, 0, result.output)
  148. self.assertEqual(result.report["restores"], 1)
  149. self.assertEqual(result.cleanup_calls, [1, 1])
  150. self.assertEqual(result.statuses[-1]["service_id"], 9001)
  151. def test_expired_session_rebuilds_then_continues_new_queries(self):
  152. result = self.run_case(["expired"])
  153. self.assertEqual(result.code, 0, result.output)
  154. self.assertEqual(result.report["restores"], 1)
  155. self.assertEqual(result.cleanup_calls, [1, 1])
  156. self.assertGreater(result.report["positiveQueries"], 0)
  157. self.assertEqual(result.instances[1].create_session.call_count, 1)
  158. def test_disconnect_recovers_but_reports_audit_gap_not_false_pass(self):
  159. result = self.run_case(["disconnect"])
  160. self.assertEqual(result.code, 6, result.output)
  161. self.assertFalse(result.report["passed"])
  162. self.assertEqual(result.report["auditGaps"], 1)
  163. self.assertEqual(result.report["restores"], 1)
  164. self.assertGreater(result.report["positiveQueries"], 0)
  165. self.assertEqual(result.cleanup_calls, [1, 1])
  166. def test_two_expirations_stop_without_third_session(self):
  167. result = self.run_case(["expired"], restored_outcomes=["expired"])
  168. self.assertEqual(result.code, 1, result.output)
  169. self.assertEqual(result.report["reason"], "consecutive-session-expirations")
  170. self.assertEqual(len(result.instances), 2)
  171. self.assertEqual(result.cleanup_calls, [1, 1])
  172. def test_transient_network_failure_retries_same_browser(self):
  173. result = self.run_case(["timeout", "available"])
  174. self.assertEqual(result.code, 0, result.output)
  175. self.assertEqual(len(result.instances), 1)
  176. self.assertGreater(result.report["positiveQueries"], 0)
  177. self.assertEqual(result.report["restores"], 0)
  178. def test_two_transient_failures_stop_without_relogin_or_third_probe(self):
  179. result = self.run_case(["timeout"])
  180. self.assertEqual(result.code, 1, result.output)
  181. self.assertEqual(result.pages[0].attempts, 2)
  182. self.assertEqual(result.report["reason"], "consecutive-calendar-network-errors")
  183. self.assertEqual(len(result.instances), 1)
  184. def test_otp_reentry_is_pending_todo_not_attempted_again(self):
  185. result = self.run_case(["expired"], restore_outcome="form")
  186. self.assertEqual(result.code, 4, result.output)
  187. self.assertEqual(result.report["state"], "needs_manual")
  188. self.assertEqual(result.report["automaticOtpTodo"], "ITA-OTP-01")
  189. self.assertIsNone(result.report["latestQuery"])
  190. self.assertEqual(result.pages[1].refresh_calls, 0)
  191. self.assertEqual(result.report["queryRequests"]["otpRequests"], 0)
  192. self.assertEqual(result.report["queryRequests"]["avantiPosts"], 0)
  193. def test_rate_limit_stops_without_account_rotation_or_relogin(self):
  194. result = self.run_case(["available", "rate"])
  195. self.assertEqual(result.code, 5, result.output)
  196. self.assertEqual(result.report["state"], "rate_limited")
  197. self.assertEqual(len(result.instances), 1)
  198. self.assertEqual(result.pages[0].attempts, 2)
  199. self.assertIsNone(result.report["latestQuery"])
  200. def test_permission_denied_stops_for_operator(self):
  201. result = self.run_case(["denied"])
  202. self.assertEqual(result.code, 4, result.output)
  203. self.assertEqual(result.report["state"], "needs_manual")
  204. self.assertEqual(result.pages[0].attempts, 1)
  205. def test_unknown_schema_stops_without_false_inventory_or_secret_log(self):
  206. result = self.run_case(["malformed"])
  207. self.assertEqual(result.code, 1, result.output)
  208. self.assertEqual(result.report["positiveQueries"], 0)
  209. self.assertEqual(result.report["negativeQueries"], 0)
  210. self.assertEqual(result.pages[0].attempts, 1)
  211. self.assertNotIn("secret-token", result.output)
  212. self.assertNotIn("private@example.test", result.output)
  213. def test_unexpected_write_request_cannot_pass(self):
  214. result = self.run_case(["unexpected-otp"])
  215. self.assertNotEqual(result.code, 0, result.output)
  216. self.assertFalse(result.report["passed"])
  217. self.assertEqual(result.report["queryRequests"]["otpRequests"], 1)
  218. self.assertEqual(result.report["positiveQueries"], 0)
  219. def test_wrong_service_after_relogin_stops_without_querying_it(self):
  220. result = self.run_case(["expired"], restore_outcome="wrong-service")
  221. self.assertEqual(result.code, 1, result.output)
  222. self.assertEqual(result.pages[1].refresh_calls, 0)
  223. self.assertFalse(result.report["passed"])
  224. def test_unverified_reservation_counts_cannot_pass(self):
  225. result = self.run_case(end_snapshot=ReservationSnapshot(1, 0))
  226. self.assertEqual(result.code, 6, result.output)
  227. self.assertFalse(result.report["passed"])
  228. self.assertFalse(result.report["reservationVerified"])
  229. def test_cancel_does_not_recreate_or_submit(self):
  230. result = self.run_case(interrupt_wait=True)
  231. self.assertEqual(result.code, 130, result.output)
  232. self.assertEqual(result.cleanup_calls, [1])
  233. self.assertFalse(result.report["passed"])
  234. def test_cancel_during_health_probe_cleans_up_without_relogin(self):
  235. result = self.run_case(interrupt_health=True)
  236. self.assertEqual(result.code, 130, result.output)
  237. self.assertEqual(result.cleanup_calls, [1])
  238. self.assertEqual(len(result.instances), 1)
  239. self.assertEqual(result.pages[0].attempts, 0)
  240. self.assertFalse(result.report["passed"])
  241. def test_cleanup_failure_is_not_reported_as_success(self):
  242. result = self.run_case(cleanup_failure=True)
  243. self.assertEqual(result.code, 3, result.output)
  244. self.assertFalse(result.report["passed"])
  245. self.assertNotIn("secret-token", result.output)
  246. def test_connect_error_reuses_existing_bounded_creation_retry(self):
  247. result = self.run_case(["expired"], create_outcomes=["connect-error", "ok"])
  248. self.assertEqual(result.code, 0, result.output)
  249. self.assertEqual(len(result.instances), 3)
  250. self.assertEqual(result.cleanup_calls, [1, 1, 1])
  251. def test_session_attempt_cap_stops_instead_of_silently_finishing(self):
  252. result = self.run_case(["expired"], max_sessions=1)
  253. self.assertEqual(result.code, 1, result.output)
  254. self.assertEqual(result.report["reason"], "max-session-attempts-reached")
  255. self.assertEqual(len(result.instances), 1)
  256. def test_failed_cleanup_during_recovery_prevents_another_browser(self):
  257. result = self.run_case(["expired"], cleanup_failure=True)
  258. self.assertEqual(result.code, 3, result.output)
  259. self.assertEqual(result.report["reason"], "cleanup-failed")
  260. self.assertEqual(len(result.instances), 1)
  261. self.assertNotIn("ITA_SOAK_CLEANUP_OK", result.output)
  262. def test_two_connection_failures_stop_using_existing_cap(self):
  263. result = self.run_case(["expired"], create_outcomes=["connect-error"])
  264. self.assertEqual(result.code, 1, result.output)
  265. self.assertEqual(result.report["reason"], "consecutive-browser-connect-failures")
  266. self.assertEqual(len(result.instances), 3) # prepared + two failed starts
  267. def test_two_disconnections_stop_without_third_browser(self):
  268. result = self.run_case(["disconnect"], restored_outcomes=["disconnect"])
  269. self.assertEqual(result.code, 1, result.output)
  270. self.assertEqual(result.report["reason"], "consecutive-browser-disconnects")
  271. self.assertEqual(len(result.instances), 2)
  272. def test_hourly_reauthentication_cap_survives_successful_queries_between_failures(self):
  273. result = self.run_case(["expired"], restored_outcomes=["available", "expired"], duration=7200)
  274. self.assertEqual(result.code, 1, result.output)
  275. self.assertEqual(result.report["reason"], "reauthentication-rate-limit")
  276. self.assertEqual(len(result.instances), 7)
  277. def test_deadline_before_successful_retry_is_incomplete_not_a_pass(self):
  278. result = self.run_case(["timeout"], duration=60)
  279. self.assertEqual(result.code, 6, result.output)
  280. self.assertEqual(result.report["positiveQueries"], 0)
  281. def test_unexercised_requested_restart_is_not_marked_verified(self):
  282. result = self.run_case(restart=100)
  283. self.assertEqual(result.code, 6, result.output)
  284. self.assertFalse(result.report["controlledRestart"])
  285. class SimpleResult:
  286. def __init__(self, code, report, statuses, output, instances, pages, cleanup_calls):
  287. self.code, self.report, self.statuses, self.output = code, report, statuses, output
  288. self.instances, self.pages, self.cleanup_calls = instances, pages, cleanup_calls
  289. class CalendarSoakCliTests(unittest.TestCase):
  290. def test_gate_transfers_the_same_prepared_instance_and_cleanup_once(self):
  291. from test import test_debug_ita_production_query_gate as workflow
  292. from tools.debug_ita_calendar_gate import CalendarGateMode
  293. def soak(_env, *, plugin, route, request_audit, before):
  294. self.assertEqual(route.service_id, 9001)
  295. self.assertEqual(before, ReservationSnapshot(0, 0))
  296. self.assertEqual(request_audit.snapshot()["otpRequests"], 1)
  297. self.assertEqual(request_audit.snapshot()["avantiPosts"], 1)
  298. plugin.page.set.auto_handle_alert.assert_called_with(accept=False)
  299. request_audit.stop()
  300. plugin.cleanup()
  301. return 0
  302. with patch("tools.debug_ita_calendar_gate.resolve_qcal_mode", return_value=CalendarGateMode.CALENDAR_SOAK), \
  303. patch("tools.debug_ita_soak.run_prepared_calendar_soak", side_effect=soak) as handoff:
  304. code, output, plugin, page, mocks, otp_reader = workflow.ProductionGateWorkflowTests()._run_gate()
  305. self.assertEqual(code, 0, output)
  306. handoff.assert_called_once()
  307. otp_reader.assert_called_once()
  308. plugin.cleanup.assert_called_once()
  309. mocks["run_production_query_acceptance"].assert_not_called()
  310. mocks["scan_calendar_horizon"].assert_not_called()
  311. def test_preflight_checks_calendar_config_without_credentials_or_browser(self):
  312. output = io.StringIO()
  313. with patch.dict("os.environ", gate_env(), clear=True), \
  314. patch("tools.debug_ita_calendar_gate.run_calendar_gate") as run, \
  315. patch("tools.debug_ita_soak._get_query_credentials") as credentials, \
  316. redirect_stdout(output):
  317. code = main(["--calendar", "--duration-seconds", "900"])
  318. self.assertEqual(code, 0)
  319. run.assert_not_called()
  320. credentials.assert_not_called()
  321. self.assertIn("external_request=false", output.getvalue())
  322. def test_unsafe_cadence_is_rejected_before_gate_or_otp(self):
  323. with patch.dict("os.environ", gate_env(), clear=True), \
  324. patch("tools.debug_ita_calendar_gate.run_calendar_gate") as run, \
  325. redirect_stderr(io.StringIO()):
  326. code = main(["--calendar", "--run", "--min-wait", "10"])
  327. self.assertEqual(code, 2)
  328. run.assert_not_called()
  329. def test_actual_gate_is_only_entered_with_run(self):
  330. with patch.dict("os.environ", gate_env(), clear=True), \
  331. patch("tools.debug_ita_calendar_gate.run_calendar_gate", return_value=0) as run:
  332. self.assertEqual(main(["--calendar", "--run", "--restart-after-probes", "1"]), 0)
  333. env = run.call_args.args[0]
  334. self.assertEqual(env["ITA_QCAL_MODE"], "calendar_soak")
  335. self.assertEqual(env["ITA_SOAK_RESTART_AFTER_PROBES"], "1")
  336. def test_limits_reject_bad_values_before_external_actions(self):
  337. for value in ("nan", "0", "86401"):
  338. with self.subTest(value=value), self.assertRaises(DebugConfigurationError):
  339. calendar_soak_limits({"ITA_SOAK_DURATION_SECONDS": value})
  340. if __name__ == "__main__":
  341. unittest.main()