test_debug_ita_soak.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558
  1. import ast
  2. import io
  3. import json
  4. import os
  5. import stat
  6. import subprocess
  7. import sys
  8. import tempfile
  9. import unittest
  10. from contextlib import redirect_stderr, redirect_stdout
  11. from pathlib import Path
  12. from unittest.mock import patch
  13. from DrissionPage.errors import BrowserConnectError, PageDisconnectedError
  14. from plugins.ita_plugin import ItaTargetServiceLinkMissingError
  15. from tools.debug_ita_session import DebugConfigurationError
  16. from tools.debug_ita_soak import (
  17. DAY_SECONDS,
  18. DEFAULT_MAX_SESSION_ATTEMPTS,
  19. DEFAULT_MAX_WAIT_SECONDS,
  20. DEFAULT_MIN_WAIT_SECONDS,
  21. run_soak,
  22. write_soak_status,
  23. )
  24. from vs_types import (
  25. AvailabilityStatus,
  26. SessionExpiredOrInvalidError,
  27. VSQueryResult,
  28. )
  29. class FakeClock:
  30. def __init__(self):
  31. self.now = 0.0
  32. def monotonic(self):
  33. return self.now
  34. def wall_time(self):
  35. return 1_800_000_000.0 + self.now
  36. def sleep(self, seconds):
  37. self.now += seconds
  38. class FakeSoakPage:
  39. url = "https://prenotami.esteri.it/Services/Booking/1321"
  40. title = "Index - Prenot@Mi"
  41. def ele(self, _selector, timeout=None):
  42. del timeout
  43. return None
  44. def run_js(self, _script):
  45. return {
  46. "path": "/Services/Booking/1321",
  47. "form": {
  48. "method": "POST",
  49. "action_path": "/Services/Booking/1321",
  50. },
  51. "field_count": 0,
  52. "fields": [],
  53. "button_count": 0,
  54. "buttons": [],
  55. }
  56. class FakeSoakPlugin:
  57. session_behaviors = [["none"]]
  58. create_behaviors = ["ok"]
  59. instances = []
  60. def __init__(self, group_id):
  61. self.group_id = group_id
  62. session_index = len(self.instances)
  63. configured_index = min(
  64. session_index,
  65. len(self.session_behaviors) - 1,
  66. )
  67. self.behaviors = list(self.session_behaviors[configured_index])
  68. create_index = min(
  69. session_index,
  70. len(self.create_behaviors) - 1,
  71. )
  72. self.create_behavior = self.create_behaviors[create_index]
  73. self.query_calls = 0
  74. self.create_session_calls = 0
  75. self.cleanup_calls = 0
  76. self.page = FakeSoakPage()
  77. self.last_booking_form_structure = None
  78. self.last_service_list_structure = None
  79. self.instances.append(self)
  80. def set_log(self, logger):
  81. self.logger = logger
  82. def set_config(self, config):
  83. self.config = config
  84. def create_session(self):
  85. self.create_session_calls += 1
  86. if self.create_behavior == "connect-error":
  87. raise BrowserConnectError("offline browser connection failed")
  88. def query(self, apt_type):
  89. index = min(self.query_calls, len(self.behaviors) - 1)
  90. behavior = self.behaviors[index]
  91. self.query_calls += 1
  92. if behavior == "missing-link":
  93. self.last_service_list_structure = {
  94. "path": "/Services",
  95. "ready_state": "complete",
  96. "table_count": 1,
  97. "row_count": 0,
  98. "booking_link_count": 0,
  99. "target_link_count": 0,
  100. }
  101. raise ItaTargetServiceLinkMissingError(
  102. "Italy target service link is missing after the bounded wait"
  103. )
  104. if behavior == "expired":
  105. raise SessionExpiredOrInvalidError("offline session expired")
  106. if behavior == "disconnected":
  107. raise PageDisconnectedError("offline browser disconnected")
  108. if behavior == "weak":
  109. return VSQueryResult(
  110. success=True,
  111. apt_type=apt_type,
  112. availability_status=AvailabilityStatus.NoneAvailable,
  113. )
  114. if behavior == "error":
  115. raise RuntimeError(
  116. "private@example.test token abcdefghijklmnopqrstuvwxyz123456"
  117. )
  118. return VSQueryResult(
  119. success=False,
  120. apt_type=apt_type,
  121. availability_status=AvailabilityStatus.NoneAvailable,
  122. )
  123. def cleanup(self):
  124. self.cleanup_calls += 1
  125. class DebugItaSoakTests(unittest.TestCase):
  126. def setUp(self):
  127. FakeSoakPlugin.session_behaviors = [["none"]]
  128. FakeSoakPlugin.create_behaviors = ["ok"]
  129. FakeSoakPlugin.instances = []
  130. @staticmethod
  131. def _run(
  132. *,
  133. duration_seconds=120,
  134. min_wait=60,
  135. max_wait=60,
  136. max_session_attempts=DEFAULT_MAX_SESSION_ATTEMPTS,
  137. restart_wait=60,
  138. sleep=None,
  139. ):
  140. clock = FakeClock()
  141. statuses = []
  142. stdout = io.StringIO()
  143. stderr = io.StringIO()
  144. if sleep is None:
  145. sleep = clock.sleep
  146. with (
  147. patch(
  148. "tools.debug_ita_soak.resolve_chrome_path",
  149. return_value=Path("/test/chrome"),
  150. ),
  151. patch(
  152. "tools.debug_ita_soak._get_query_credentials",
  153. return_value=("query@example.test", "query-password"),
  154. ) as credential_reader,
  155. patch(
  156. "tools.debug_ita_soak._prepare_query_profile",
  157. return_value=Path("/offline/query-profile"),
  158. ) as profile_preparer,
  159. patch.dict("tools.debug_ita_soak.os.environ", {}, clear=False),
  160. redirect_stdout(stdout),
  161. redirect_stderr(stderr),
  162. ):
  163. exit_code = run_soak(
  164. {},
  165. apply=True,
  166. non_interactive=True,
  167. duration_seconds=duration_seconds,
  168. min_wait=min_wait,
  169. max_wait=max_wait,
  170. max_session_attempts=max_session_attempts,
  171. restart_wait=restart_wait,
  172. randint=lambda _minimum, _maximum: max_wait,
  173. sleep=sleep,
  174. monotonic=clock.monotonic,
  175. wall_time=clock.wall_time,
  176. plugin_factory=FakeSoakPlugin,
  177. status_writer=lambda body: statuses.append(dict(body)),
  178. )
  179. return (
  180. exit_code,
  181. stdout.getvalue(),
  182. stderr.getvalue(),
  183. clock,
  184. statuses,
  185. credential_reader,
  186. profile_preparer,
  187. )
  188. def test_accelerated_full_day_uses_one_session_and_completes(self):
  189. result = self._run(
  190. duration_seconds=DAY_SECONDS,
  191. min_wait=DEFAULT_MAX_WAIT_SECONDS,
  192. max_wait=DEFAULT_MAX_WAIT_SECONDS,
  193. )
  194. exit_code, stdout, stderr, clock, statuses, _, _ = result
  195. plugin = FakeSoakPlugin.instances[0]
  196. self.assertEqual(exit_code, 0)
  197. self.assertEqual(clock.now, DAY_SECONDS)
  198. self.assertEqual(len(FakeSoakPlugin.instances), 1)
  199. self.assertEqual(plugin.create_session_calls, 1)
  200. self.assertEqual(plugin.query_calls, 288)
  201. self.assertEqual(plugin.cleanup_calls, 1)
  202. self.assertIn(
  203. "ITA_SOAK_COMPLETED duration_seconds=86400 probes=288",
  204. stdout,
  205. )
  206. self.assertIn("ITA_SOAK_CLEANUP_OK", stdout)
  207. self.assertEqual(statuses[-1]["state"], "completed")
  208. self.assertEqual(statuses[-1]["elapsed_seconds"], DAY_SECONDS)
  209. self.assertEqual(statuses[-1]["exit_code"], 0)
  210. self.assertEqual(stderr, "")
  211. def test_accelerated_full_day_survives_many_normal_session_expirations(self):
  212. FakeSoakPlugin.session_behaviors = [
  213. ["none", "none", "expired"] for _ in range(200)
  214. ]
  215. result = self._run(
  216. duration_seconds=DAY_SECONDS,
  217. min_wait=DEFAULT_MAX_WAIT_SECONDS,
  218. max_wait=DEFAULT_MAX_WAIT_SECONDS,
  219. )
  220. exit_code, stdout, stderr, clock, statuses, _, _ = result
  221. self.assertEqual(exit_code, 0)
  222. self.assertEqual(clock.now, DAY_SECONDS)
  223. self.assertGreaterEqual(statuses[-1]["session_expirations"], 96)
  224. self.assertGreaterEqual(statuses[-1]["session_attempts"], 97)
  225. self.assertEqual(
  226. statuses[-1]["consecutive_session_expirations"],
  227. 0,
  228. )
  229. self.assertEqual(statuses[-1]["state"], "completed")
  230. self.assertIn("ITA_SOAK_COMPLETED duration_seconds=86400", stdout)
  231. self.assertNotIn("max-session-attempts-reached", stdout)
  232. self.assertEqual(stderr, "")
  233. def test_one_missing_link_is_recorded_then_next_probe_can_recover(self):
  234. FakeSoakPlugin.session_behaviors = [["missing-link", "none"]]
  235. exit_code, stdout, stderr, _, statuses, _, _ = self._run()
  236. self.assertEqual(exit_code, 0)
  237. self.assertEqual(FakeSoakPlugin.instances[0].query_calls, 2)
  238. self.assertIn("ITA_SOAK_TRANSIENT", stdout)
  239. self.assertIn("consecutive=1 total=1", stdout)
  240. self.assertIn("ITA_SERVICE_LIST_STRUCTURE=", stdout)
  241. self.assertEqual(statuses[-1]["transient_errors"], 1)
  242. self.assertEqual(stderr, "")
  243. def test_second_consecutive_missing_link_stops_without_third_query(self):
  244. FakeSoakPlugin.session_behaviors = [
  245. ["missing-link", "missing-link", "none"]
  246. ]
  247. exit_code, stdout, stderr, _, statuses, _, _ = self._run(
  248. duration_seconds=DAY_SECONDS
  249. )
  250. self.assertEqual(exit_code, 1)
  251. self.assertEqual(FakeSoakPlugin.instances[0].query_calls, 2)
  252. self.assertIn(
  253. "ITA_SOAK_FAILED reason=consecutive-target-link-misses",
  254. stdout,
  255. )
  256. self.assertEqual(statuses[-1]["state"], "failed")
  257. self.assertEqual(statuses[-1]["exit_code"], 1)
  258. self.assertEqual(stderr, "")
  259. def test_expired_session_is_cleaned_then_recreated_after_safe_wait(self):
  260. FakeSoakPlugin.session_behaviors = [["expired"], ["none"]]
  261. exit_code, stdout, stderr, clock, statuses, _, _ = self._run()
  262. self.assertEqual(exit_code, 0)
  263. self.assertEqual(len(FakeSoakPlugin.instances), 2)
  264. self.assertEqual(FakeSoakPlugin.instances[0].cleanup_calls, 1)
  265. self.assertEqual(FakeSoakPlugin.instances[1].cleanup_calls, 1)
  266. self.assertEqual(clock.now, 120)
  267. self.assertIn("ITA_SOAK_SESSION_EXPIRED", stdout)
  268. self.assertIn("session_attempts=2", stdout)
  269. self.assertEqual(statuses[-1]["session_attempts"], 2)
  270. self.assertEqual(stderr, "")
  271. def test_second_consecutive_session_expiration_stops_without_third_session(self):
  272. FakeSoakPlugin.session_behaviors = [
  273. ["expired"],
  274. ["expired"],
  275. ["none"],
  276. ]
  277. exit_code, stdout, stderr, _, statuses, _, _ = self._run(
  278. duration_seconds=DAY_SECONDS
  279. )
  280. self.assertEqual(exit_code, 1)
  281. self.assertEqual(len(FakeSoakPlugin.instances), 2)
  282. self.assertIn(
  283. "ITA_SOAK_FAILED reason=consecutive-session-expirations",
  284. stdout,
  285. )
  286. self.assertEqual(statuses[-1]["session_expirations"], 2)
  287. self.assertEqual(
  288. statuses[-1]["consecutive_session_expirations"],
  289. 2,
  290. )
  291. self.assertEqual(statuses[-1]["state"], "failed")
  292. self.assertEqual(stderr, "")
  293. def test_reauthentication_rate_limit_stops_dense_login_loop(self):
  294. FakeSoakPlugin.session_behaviors = [
  295. ["none", "expired"] for _ in range(10)
  296. ]
  297. exit_code, stdout, stderr, _, statuses, _, _ = self._run(
  298. duration_seconds=DAY_SECONDS
  299. )
  300. self.assertEqual(exit_code, 1)
  301. self.assertIn(
  302. "ITA_SOAK_FAILED reason=reauthentication-rate-limit",
  303. stdout,
  304. )
  305. self.assertEqual(statuses[-1]["session_expirations"], 7)
  306. self.assertEqual(
  307. statuses[-1]["session_reauthentications_last_hour"],
  308. 7,
  309. )
  310. self.assertEqual(statuses[-1]["state"], "failed")
  311. self.assertEqual(stderr, "")
  312. def test_disconnected_browser_is_cleaned_then_recreated_after_safe_wait(self):
  313. FakeSoakPlugin.session_behaviors = [["disconnected"], ["none"]]
  314. exit_code, stdout, stderr, clock, statuses, _, _ = self._run()
  315. self.assertEqual(exit_code, 0)
  316. self.assertEqual(len(FakeSoakPlugin.instances), 2)
  317. self.assertEqual(FakeSoakPlugin.instances[0].cleanup_calls, 1)
  318. self.assertEqual(FakeSoakPlugin.instances[1].cleanup_calls, 1)
  319. self.assertEqual(clock.now, 120)
  320. self.assertIn("ITA_SOAK_BROWSER_DISCONNECTED", stdout)
  321. self.assertIn("consecutive=1 total=1", stdout)
  322. self.assertEqual(statuses[-1]["browser_disconnects"], 1)
  323. self.assertEqual(statuses[-1]["session_attempts"], 2)
  324. self.assertEqual(stderr, "")
  325. def test_second_consecutive_browser_disconnect_stops_without_third_session(self):
  326. FakeSoakPlugin.session_behaviors = [
  327. ["disconnected"],
  328. ["disconnected"],
  329. ["none"],
  330. ]
  331. exit_code, stdout, stderr, _, statuses, _, _ = self._run(
  332. duration_seconds=DAY_SECONDS
  333. )
  334. self.assertEqual(exit_code, 1)
  335. self.assertEqual(len(FakeSoakPlugin.instances), 2)
  336. self.assertEqual(FakeSoakPlugin.instances[0].cleanup_calls, 1)
  337. self.assertEqual(FakeSoakPlugin.instances[1].cleanup_calls, 1)
  338. self.assertIn(
  339. "ITA_SOAK_FAILED reason=consecutive-browser-disconnects",
  340. stdout,
  341. )
  342. self.assertEqual(statuses[-1]["browser_disconnects"], 2)
  343. self.assertEqual(statuses[-1]["state"], "failed")
  344. self.assertEqual(statuses[-1]["exit_code"], 1)
  345. self.assertEqual(stderr, "")
  346. def test_one_browser_connect_failure_is_retried_after_safe_wait(self):
  347. FakeSoakPlugin.create_behaviors = ["connect-error", "ok"]
  348. FakeSoakPlugin.session_behaviors = [["none"], ["none"]]
  349. exit_code, stdout, stderr, clock, statuses, _, _ = self._run()
  350. self.assertEqual(exit_code, 0)
  351. self.assertEqual(len(FakeSoakPlugin.instances), 2)
  352. self.assertEqual(FakeSoakPlugin.instances[0].cleanup_calls, 1)
  353. self.assertEqual(FakeSoakPlugin.instances[1].cleanup_calls, 1)
  354. self.assertEqual(clock.now, 120)
  355. self.assertIn("ITA_SOAK_BROWSER_CONNECT_FAILED", stdout)
  356. self.assertIn("consecutive=1 total=1", stdout)
  357. self.assertEqual(statuses[-1]["browser_connect_failures"], 1)
  358. self.assertEqual(statuses[-1]["session_attempts"], 2)
  359. self.assertEqual(stderr, "")
  360. def test_second_browser_connect_failure_stops_without_third_attempt(self):
  361. FakeSoakPlugin.create_behaviors = [
  362. "connect-error",
  363. "connect-error",
  364. "ok",
  365. ]
  366. exit_code, stdout, stderr, _, statuses, _, _ = self._run(
  367. duration_seconds=DAY_SECONDS
  368. )
  369. self.assertEqual(exit_code, 1)
  370. self.assertEqual(len(FakeSoakPlugin.instances), 2)
  371. self.assertIn(
  372. "ITA_SOAK_FAILED reason=consecutive-browser-connect-failures",
  373. stdout,
  374. )
  375. self.assertEqual(statuses[-1]["browser_connect_failures"], 2)
  376. self.assertEqual(statuses[-1]["state"], "failed")
  377. self.assertEqual(statuses[-1]["exit_code"], 1)
  378. self.assertEqual(stderr, "")
  379. def test_weak_signal_stops_immediately_without_waiting_or_booking(self):
  380. FakeSoakPlugin.session_behaviors = [["weak", "none"]]
  381. exit_code, stdout, stderr, clock, statuses, _, _ = self._run(
  382. duration_seconds=DAY_SECONDS
  383. )
  384. self.assertEqual(exit_code, 4)
  385. self.assertEqual(clock.now, 0)
  386. self.assertEqual(FakeSoakPlugin.instances[0].query_calls, 1)
  387. self.assertIn("ITA_SOAK_WEAK_SIGNAL", stdout)
  388. self.assertEqual(statuses[-1]["state"], "weak_signal")
  389. self.assertEqual(statuses[-1]["exit_code"], 4)
  390. self.assertEqual(stderr, "")
  391. def test_default_preflight_reads_no_credentials_and_makes_no_session(self):
  392. stdout = io.StringIO()
  393. stderr = io.StringIO()
  394. with (
  395. patch(
  396. "tools.debug_ita_soak._get_query_credentials",
  397. side_effect=AssertionError("credentials must not be read"),
  398. ),
  399. redirect_stdout(stdout),
  400. redirect_stderr(stderr),
  401. ):
  402. exit_code = run_soak(
  403. {},
  404. apply=False,
  405. non_interactive=True,
  406. duration_seconds=DAY_SECONDS,
  407. min_wait=DEFAULT_MIN_WAIT_SECONDS,
  408. max_wait=DEFAULT_MAX_WAIT_SECONDS,
  409. )
  410. self.assertEqual(exit_code, 0)
  411. self.assertEqual(FakeSoakPlugin.instances, [])
  412. self.assertIn("ITA_SOAK_PREFLIGHT_OK", stdout.getvalue())
  413. self.assertIn("external_request=false", stdout.getvalue())
  414. self.assertEqual(stderr.getvalue(), "")
  415. def test_duration_over_one_day_is_rejected_before_credentials(self):
  416. with (
  417. patch(
  418. "tools.debug_ita_soak._get_query_credentials",
  419. side_effect=AssertionError("credentials must not be read"),
  420. ),
  421. self.assertRaises(DebugConfigurationError),
  422. ):
  423. run_soak(
  424. {},
  425. apply=True,
  426. non_interactive=True,
  427. duration_seconds=DAY_SECONDS + 1,
  428. min_wait=DEFAULT_MIN_WAIT_SECONDS,
  429. max_wait=DEFAULT_MAX_WAIT_SECONDS,
  430. )
  431. def test_status_file_is_private_and_contains_no_credentials(self):
  432. with tempfile.TemporaryDirectory() as temp_dir:
  433. status_path = Path(temp_dir) / "ita_soak" / "status.json"
  434. write_soak_status(
  435. {
  436. "state": "running",
  437. "service_id": 1321,
  438. "elapsed_seconds": 60,
  439. "probes": 1,
  440. },
  441. status_path=status_path,
  442. )
  443. body = status_path.read_text(encoding="utf-8")
  444. self.assertEqual(json.loads(body)["state"], "running")
  445. self.assertNotIn("username", body.lower())
  446. self.assertNotIn("password", body.lower())
  447. self.assertNotIn("cookie", body.lower())
  448. if os.name == "posix":
  449. self.assertEqual(
  450. stat.S_IMODE(status_path.parent.stat().st_mode),
  451. 0o700,
  452. )
  453. self.assertEqual(
  454. stat.S_IMODE(status_path.stat().st_mode),
  455. 0o600,
  456. )
  457. def test_cli_default_is_zero_network_preflight(self):
  458. repo_root = Path(__file__).resolve().parents[1]
  459. completed = subprocess.run(
  460. [sys.executable, str(repo_root / "tools" / "debug_ita_soak.py")],
  461. cwd=repo_root,
  462. check=False,
  463. capture_output=True,
  464. text=True,
  465. )
  466. self.assertEqual(completed.returncode, 0)
  467. self.assertIn("ITA_SOAK_PREFLIGHT_OK", completed.stdout)
  468. self.assertIn("duration_seconds=86400", completed.stdout)
  469. self.assertIn("external_request=false", completed.stdout)
  470. self.assertEqual(completed.stderr, "")
  471. def test_source_has_one_query_call_and_no_book_call(self):
  472. source_path = (
  473. Path(__file__).resolve().parents[1] / "tools" / "debug_ita_soak.py"
  474. )
  475. tree = ast.parse(source_path.read_text(encoding="utf-8"))
  476. query_calls = []
  477. book_calls = []
  478. for node in ast.walk(tree):
  479. if not isinstance(node, ast.Call):
  480. continue
  481. if not isinstance(node.func, ast.Attribute):
  482. continue
  483. if node.func.attr == "query":
  484. query_calls.append(node)
  485. if node.func.attr == "book":
  486. book_calls.append(node)
  487. self.assertEqual(len(query_calls), 1)
  488. self.assertEqual(book_calls, [])
  489. if __name__ == "__main__":
  490. unittest.main()