"""Offline regressions for order leases, durable results and browser scheduling.""" import json import tempfile import threading import time import unittest from pathlib import Path from unittest.mock import MagicMock, patch from booker import BuiltinBookerGCO, OrderBookerGCO from sentinel import SentinelGCO from core_types import (AppointmentType, AvailabilityStatus, BookResult, GroupConfig, AccountIncompleteError, PluginConfig, QueryResult, RateLimiteddError, Task) from main_sweeper import OrphanTaskSweeper from toolkit.task_executor import TaskExecutor from utils.booking_outbox import BookingOutbox from utils.safe_redis_cli import SafeRedisClient class SchedulerTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.cloud = MagicMock() self.cloud_patch = patch('toolkit.cloud_api.CloudApi.Instance', return_value=self.cloud) self.cloud_patch.start() self.addCleanup(self.cloud_patch.stop) def make(self, cls=BuiltinBookerGCO): g = cls(GroupConfig(identifier='audit', enable=True), {}, logger=lambda _: None) g.redis_client = MagicMock() g.redis_client.get.return_value = None if hasattr(g, '_outbox'): g._outbox = BookingOutbox(str(Path(self.tmp.name) / 'outbox.sqlite3')) self.addCleanup(g.stop) return g def signal(self, key): return json.dumps({'apt_type': {'routing_key': key}, 'query_result': {'availability_status': 'Available'}}) def test_long_builtin_booking_renews_order_lease_without_browser_ping(self): g = self.make() plugin = MagicMock(config=PluginConfig()) task = Task(instance=plugin) self.cloud.get_vas_task_pop.return_value = {'id': 42} clock = [1000.0] def book(*_): clock[0] += 131 g._renew_leases() mapping = g.redis_client.bulk_zadd.call_args.args[1] self.assertEqual(mapping['42'], 1251.0) plugin.keep_alive.assert_not_called() return BookResult() plugin.book.side_effect = book with patch('booker.time.time', side_effect=lambda: clock[0]): g._execute_book_job(task, QueryResult()) self.assertNotIn('42', g._leased_orders) def test_spawn_rate_limit_retains_recovery_deadline(self): g = self.make(OrderBookerGCO) self.cloud.get_vas_task_pop.return_value = {'id': 42, 'user_inputs': {'username': 'audit'}} plugin = MagicMock() plugin.create_session.side_effect = RateLimiteddError() g.m_factory.create = MagicMock(return_value=plugin) g._spawn_worker('a') deadline = g.redis_client.zadd.call_args.args[1]['42'] self.assertGreaterEqual(deadline, g.m_spawn_cooldown_until) g.redis_client.zrem.assert_not_called() self.assertNotIn('42', g._leased_orders) plugin.cleanup.assert_called_once() def test_failed_success_sync_is_durable_and_retryable(self): g = self.make(OrderBookerGCO) self.cloud.update_vas_task.side_effect = RuntimeError('offline') with patch('booker.notify_book_result'): g._push_success_notification(42, 'order', BookResult(success=True, urn='confirmed')) g.redis_client._client.zrem.assert_not_called() reopened = BookingOutbox(g._outbox.path) self.assertEqual(reopened.get(42)['grabbed_history']['urn'], 'confirmed') self.cloud.update_vas_task.side_effect = None with patch('utils.booking_outbox.time.time', return_value=time.time() + 10): reopened.flush(self.cloud, g.redis_client._client, lambda _: None) g.redis_client._client.zrem.assert_called_once_with(g.m_tracker_key, '42') self.assertIsNone(reopened.get(42)) def test_redis_failure_keeps_success_on_disk(self): outbox = BookingOutbox(str(Path(self.tmp.name) / 'redis-failure.sqlite3')) outbox.put(42, 'tracker', {'status': 'grabbed'}) redis = MagicMock() redis.set.side_effect = RuntimeError('redis offline') outbox.flush(self.cloud, redis, lambda _: None) self.assertIsNotNone(outbox.get(42)) redis.zrem.assert_not_called() self.cloud.update_vas_task.assert_not_called() def test_two_flushers_claim_each_success_only_once(self): outbox = BookingOutbox(str(Path(self.tmp.name) / 'concurrent.sqlite3')) outbox.put(42, 'tracker', {'status': 'grabbed'}) entered = threading.Event() release = threading.Event() self.cloud.update_vas_task.side_effect = lambda *_: (entered.set(), release.wait(3)) first = threading.Thread(target=outbox.flush, args=(self.cloud, MagicMock(), lambda _: None)) first.start() try: self.assertTrue(entered.wait(2)) reopened = BookingOutbox(outbox.path) reopened.flush(self.cloud, MagicMock(), lambda _: None) self.cloud.update_vas_task.assert_called_once() finally: release.set() first.join(3) self.assertFalse(first.is_alive()) self.assertIsNone(outbox.get(42)) def test_failed_terminal_status_update_retains_recovery_tracker(self): g = self.make(OrderBookerGCO) self.cloud.update_vas_task.side_effect = RuntimeError('cloud offline') g._dispatch_error_policy('spawn', AccountIncompleteError(), plg_cfg=PluginConfig(), task_id=42) g.redis_client.zrem.assert_not_called() def test_cleanup_retry_does_not_repeat_confirmed_cloud_update(self): outbox = BookingOutbox(str(Path(self.tmp.name) / 'cleanup.sqlite3')) outbox.put(42, 'tracker', {'status': 'grabbed'}) redis = MagicMock() redis.zrem.side_effect = RuntimeError('redis offline') outbox.flush(self.cloud, redis, lambda _: None) self.assertIsNotNone(outbox.get(42)) redis.zrem.side_effect = None with patch('utils.booking_outbox.time.time', return_value=time.time() + 10): outbox.flush(self.cloud, redis, lambda _: None) self.cloud.update_vas_task.assert_called_once() self.assertIsNone(outbox.get(42)) def test_storage_outage_retains_in_memory_success_and_lease(self): g = self.make() g._leased_orders.add('42') g.redis_client.set.return_value = False with patch.object(g._outbox, 'put', side_effect=OSError('disk unavailable')), \ patch('booker.notify_book_result'): g._push_success_notification(42, 'order', BookResult(success=True)) self.assertIn('42', g._pending_results) self.assertIn('42', g._leased_orders) g.redis_client.set.return_value = True g._sync_successes() self.assertNotIn('42', g._pending_results) self.assertNotIn('42', g._leased_orders) def test_slow_route_does_not_block_other_route_or_duplicate_booking(self): g = self.make() g.m_cfg.appointment_types = [AppointmentType(routing_key=k) for k in ('a', 'b')] g.m_tasks = [Task(instance=MagicMock(), acceptable_routing_keys=[k]) for k in ('a', 'b')] g.redis_client.mget.return_value = [self.signal('a'), self.signal('b')] entered = {k: threading.Event() for k in ('a', 'b')} release = threading.Event() calls = [] def job(task, result): key = result.apt_type.routing_key calls.append(key) entered[key].set() release.wait(3) g._execute_book_job = job try: g._poll_booking_signals() self.assertTrue(entered['a'].wait(2)) self.assertTrue(entered['b'].wait(2)) g._schedule_maintenance() for task in g.m_tasks: task.instance.keep_alive.assert_not_called() g._poll_booking_signals() self.assertCountEqual(calls, ['a', 'b']) self.assertEqual(g.redis_client.mget.call_count, 1) finally: release.set() def test_busy_browser_is_not_refreshed_by_maintenance(self): g = self.make() plugin = MagicMock() task = Task(instance=plugin, busy=True) g.m_tasks = [task] g._schedule_maintenance() plugin.keep_alive.assert_not_called() def test_cleanup_waits_for_inflight_booking(self): g = self.make() plugin = MagicMock() task = Task(instance=plugin, busy=True) g.m_tasks = [task] started = threading.Event() release = threading.Event() g._execute_book_job = lambda *_: (started.set(), release.wait(3)) g._executor.submit(g._run_book_job, task, QueryResult()) self.assertTrue(started.wait(2)) stopper = threading.Thread(target=g.stop) stopper.start() try: self.assertTrue(g.m_stop_event.wait(2)) plugin.cleanup.assert_not_called() finally: release.set() stopper.join(3) self.assertFalse(stopper.is_alive()) plugin.cleanup.assert_called_once() def test_spawn_finishing_after_stop_is_cleaned_in_all_pool_modes(self): for cls, method in ((SentinelGCO, '_spawn_sentinel_worker'), (BuiltinBookerGCO, '_spawn_worker'), (OrderBookerGCO, '_spawn_worker')): with self.subTest(cls=cls): g = self.make(cls) self.cloud.get_vas_task_pop.return_value = {'id': 42, 'user_inputs': {'username': 'audit'}} plugin = MagicMock() plugin.create_session.side_effect = g.stop g.m_factory.create = MagicMock(return_value=plugin) getattr(g, method)(*(['a'] if cls is OrderBookerGCO else [])) self.assertEqual(g.m_tasks, []) # Forced shutdown and late creator completion may both clean; # cleanup is idempotent and ownership must be fully released. plugin.cleanup.assert_called() self.assertEqual(g._resources._entries, {}) def test_empty_group_does_not_poll_redis(self): g = self.make() g.m_cfg.appointment_types = [AppointmentType(routing_key='a')] g._poll_booking_signals() g.redis_client.mget.assert_not_called() def test_signal_reads_are_batched(self): g = self.make() keys = ['a', 'b', 'c'] g.m_cfg.appointment_types = [AppointmentType(routing_key=k) for k in keys] g.m_tasks = [Task(instance=MagicMock(), acceptable_routing_keys=keys)] g.redis_client.mget.return_value = [None] * 3 g._poll_booking_signals() g.redis_client.mget.assert_called_once_with(['vs:signal:' + k for k in keys]) g.redis_client.get.assert_not_called() def test_available_signal_without_date_does_not_crash_or_book(self): g = self.make(OrderBookerGCO) result = QueryResult(availability_status=AvailabilityStatus.Available) self.assertFalse(g._is_date_of_interest(Task(instance=MagicMock()), result)) def test_sentinel_slow_health_check_does_not_stall_other_instances(self): g = self.make(SentinelGCO) g.m_cfg.appointment_types = [AppointmentType(routing_key='a')] slow = MagicMock(config=PluginConfig()) fast = MagicMock(config=PluginConfig()) entered = threading.Event() release = threading.Event() queried = threading.Event() slow.health_check.side_effect = lambda: (entered.set(), release.wait(3), True)[-1] slow.query.return_value = QueryResult() fast.query.side_effect = lambda *_: (queried.set(), QueryResult())[-1] g.m_tasks = [Task(instance=slow), Task(instance=fast)] try: with patch('sentinel.time.time', return_value=1000): g._schedule_monitor_jobs() self.assertTrue(entered.wait(2)) with patch('sentinel.time.time', return_value=1001): deadline = time.monotonic() + 1 while g.m_tasks[1].busy and time.monotonic() < deadline: time.sleep(0.001) g._schedule_monitor_jobs() self.assertTrue(queried.wait(2)) finally: release.set() def test_executor_rejects_overload_instead_of_queuing(self): executor = TaskExecutor(1, 'audit') release = threading.Event() try: self.assertTrue(executor.submit(release.wait, 2)) self.assertFalse(executor.submit(lambda: None)) finally: release.set() executor.shutdown() class SweeperTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.sweeper = OrphanTaskSweeper({}, logger=MagicMock()) self.sweeper.redis_client = MagicMock() self.sweeper._outbox = MagicMock() self.sweeper._outbox.get.return_value = None self.sweeper.redis_client.get.return_value = None self.sweeper.redis_client.zscore.return_value = 0 self.cloud = MagicMock() self.cloud.get_vas_task.return_value = {'status': 'processing'} patcher = patch('toolkit.cloud_api.CloudApi.Instance', return_value=self.cloud) patcher.start() self.addCleanup(patcher.stop) def test_scan_instead_of_keys(self): r = self.sweeper.redis_client r.scan_iter.return_value = [b'tracker'] r.zrangebyscore.return_value = [b'42'] with patch.object(self.sweeper, '_rescue_task') as rescue: self.sweeper._sweep() r.keys.assert_not_called() r.scan_iter.assert_called_once_with(match='vs:worker:tasks_tracker:*', count=100) rescue.assert_called_once_with('tracker', 42) def test_pending_success_is_never_requeued(self): self.sweeper._outbox.get.return_value = {'status': 'grabbed'} self.sweeper._rescue_task('tracker', 42) self.cloud.return_vas_task_to_queue.assert_not_called() def test_remote_success_marker_is_synchronized(self): self.sweeper.redis_client.get.return_value = '{"status": "grabbed"}' self.sweeper._rescue_task('tracker', 42) self.sweeper._outbox.put.assert_called_once_with(42, 'tracker', {'status': 'grabbed'}) self.sweeper._outbox.flush.assert_called_once() self.cloud.return_vas_task_to_queue.assert_not_called() def test_lease_renewed_during_cloud_read_is_not_requeued(self): self.sweeper.redis_client.zscore.side_effect = [0, time.time() + 120] self.sweeper._rescue_task('tracker', 42) self.cloud.return_vas_task_to_queue.assert_not_called() def test_terminal_failed_order_is_not_requeued(self): self.cloud.get_vas_task.return_value = {'status': 'failed'} self.sweeper._rescue_task('tracker', 42) self.cloud.return_vas_task_to_queue.assert_not_called() self.sweeper.redis_client.zrem.assert_called_once_with('tracker', 42) class RedisTests(unittest.TestCase): def test_heartbeat_uses_one_zadd_for_entire_mapping(self): client = SafeRedisClient({}) client._client = MagicMock() mapping = {str(i): 123 for i in range(100)} self.assertTrue(client.bulk_zadd('tracker', mapping)) client._client.zadd.assert_called_once_with('tracker', mapping) client._client.pipeline.assert_not_called() if __name__ == '__main__': unittest.main()