| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344 |
- """Offline regressions for order leases, durable results and browser scheduling."""
- import json
- import tempfile
- import threading
- import time
- import unittest
- from pathlib import Path
- from unittest.mock import MagicMock, patch
- from booker import BuiltinBookerGCO, OrderBookerGCO
- from sentinel import SentinelGCO
- from core_types import (AppointmentType, AvailabilityStatus, BookResult, GroupConfig,
- AccountIncompleteError, PluginConfig, QueryResult, RateLimiteddError, Task)
- from main_sweeper import OrphanTaskSweeper
- from toolkit.task_executor import TaskExecutor
- from utils.booking_outbox import BookingOutbox
- from utils.safe_redis_cli import SafeRedisClient
- class SchedulerTests(unittest.TestCase):
- def setUp(self):
- self.tmp = tempfile.TemporaryDirectory()
- self.addCleanup(self.tmp.cleanup)
- self.cloud = MagicMock()
- self.cloud_patch = patch('toolkit.cloud_api.CloudApi.Instance', return_value=self.cloud)
- self.cloud_patch.start()
- self.addCleanup(self.cloud_patch.stop)
- def make(self, cls=BuiltinBookerGCO):
- g = cls(GroupConfig(identifier='audit', enable=True), {}, logger=lambda _: None)
- g.redis_client = MagicMock()
- g.redis_client.get.return_value = None
- if hasattr(g, '_outbox'):
- g._outbox = BookingOutbox(str(Path(self.tmp.name) / 'outbox.sqlite3'))
- self.addCleanup(g.stop)
- return g
- def signal(self, key):
- return json.dumps({'apt_type': {'routing_key': key}, 'query_result': {'availability_status': 'Available'}})
- def test_long_builtin_booking_renews_order_lease_without_browser_ping(self):
- g = self.make()
- plugin = MagicMock(config=PluginConfig())
- task = Task(instance=plugin)
- self.cloud.get_vas_task_pop.return_value = {'id': 42}
- clock = [1000.0]
- def book(*_):
- clock[0] += 131
- g._renew_leases()
- mapping = g.redis_client.bulk_zadd.call_args.args[1]
- self.assertEqual(mapping['42'], 1251.0)
- plugin.keep_alive.assert_not_called()
- return BookResult()
- plugin.book.side_effect = book
- with patch('booker.time.time', side_effect=lambda: clock[0]):
- g._execute_book_job(task, QueryResult())
- self.assertNotIn('42', g._leased_orders)
- def test_spawn_rate_limit_retains_recovery_deadline(self):
- g = self.make(OrderBookerGCO)
- self.cloud.get_vas_task_pop.return_value = {'id': 42, 'user_inputs': {'username': 'audit'}}
- plugin = MagicMock()
- plugin.create_session.side_effect = RateLimiteddError()
- g.m_factory.create = MagicMock(return_value=plugin)
- g._spawn_worker('a')
- deadline = g.redis_client.zadd.call_args.args[1]['42']
- self.assertGreaterEqual(deadline, g.m_spawn_cooldown_until)
- g.redis_client.zrem.assert_not_called()
- self.assertNotIn('42', g._leased_orders)
- plugin.cleanup.assert_called_once()
- def test_failed_success_sync_is_durable_and_retryable(self):
- g = self.make(OrderBookerGCO)
- self.cloud.update_vas_task.side_effect = RuntimeError('offline')
- with patch('booker.notify_book_result'):
- g._push_success_notification(42, 'order', BookResult(success=True, urn='confirmed'))
- g.redis_client._client.zrem.assert_not_called()
- reopened = BookingOutbox(g._outbox.path)
- self.assertEqual(reopened.get(42)['grabbed_history']['urn'], 'confirmed')
- self.cloud.update_vas_task.side_effect = None
- with patch('utils.booking_outbox.time.time', return_value=time.time() + 10):
- reopened.flush(self.cloud, g.redis_client._client, lambda _: None)
- g.redis_client._client.zrem.assert_called_once_with(g.m_tracker_key, '42')
- self.assertIsNone(reopened.get(42))
- def test_redis_failure_keeps_success_on_disk(self):
- outbox = BookingOutbox(str(Path(self.tmp.name) / 'redis-failure.sqlite3'))
- outbox.put(42, 'tracker', {'status': 'grabbed'})
- redis = MagicMock()
- redis.set.side_effect = RuntimeError('redis offline')
- outbox.flush(self.cloud, redis, lambda _: None)
- self.assertIsNotNone(outbox.get(42))
- redis.zrem.assert_not_called()
- self.cloud.update_vas_task.assert_not_called()
- def test_two_flushers_claim_each_success_only_once(self):
- outbox = BookingOutbox(str(Path(self.tmp.name) / 'concurrent.sqlite3'))
- outbox.put(42, 'tracker', {'status': 'grabbed'})
- entered = threading.Event()
- release = threading.Event()
- self.cloud.update_vas_task.side_effect = lambda *_: (entered.set(), release.wait(3))
- first = threading.Thread(target=outbox.flush, args=(self.cloud, MagicMock(), lambda _: None))
- first.start()
- try:
- self.assertTrue(entered.wait(2))
- reopened = BookingOutbox(outbox.path)
- reopened.flush(self.cloud, MagicMock(), lambda _: None)
- self.cloud.update_vas_task.assert_called_once()
- finally:
- release.set()
- first.join(3)
- self.assertFalse(first.is_alive())
- self.assertIsNone(outbox.get(42))
- def test_failed_terminal_status_update_retains_recovery_tracker(self):
- g = self.make(OrderBookerGCO)
- self.cloud.update_vas_task.side_effect = RuntimeError('cloud offline')
- g._dispatch_error_policy('spawn', AccountIncompleteError(),
- plg_cfg=PluginConfig(), task_id=42)
- g.redis_client.zrem.assert_not_called()
- def test_cleanup_retry_does_not_repeat_confirmed_cloud_update(self):
- outbox = BookingOutbox(str(Path(self.tmp.name) / 'cleanup.sqlite3'))
- outbox.put(42, 'tracker', {'status': 'grabbed'})
- redis = MagicMock()
- redis.zrem.side_effect = RuntimeError('redis offline')
- outbox.flush(self.cloud, redis, lambda _: None)
- self.assertIsNotNone(outbox.get(42))
- redis.zrem.side_effect = None
- with patch('utils.booking_outbox.time.time', return_value=time.time() + 10):
- outbox.flush(self.cloud, redis, lambda _: None)
- self.cloud.update_vas_task.assert_called_once()
- self.assertIsNone(outbox.get(42))
- def test_storage_outage_retains_in_memory_success_and_lease(self):
- g = self.make()
- g._leased_orders.add('42')
- g.redis_client.set.return_value = False
- with patch.object(g._outbox, 'put', side_effect=OSError('disk unavailable')), \
- patch('booker.notify_book_result'):
- g._push_success_notification(42, 'order', BookResult(success=True))
- self.assertIn('42', g._pending_results)
- self.assertIn('42', g._leased_orders)
- g.redis_client.set.return_value = True
- g._sync_successes()
- self.assertNotIn('42', g._pending_results)
- self.assertNotIn('42', g._leased_orders)
- def test_slow_route_does_not_block_other_route_or_duplicate_booking(self):
- g = self.make()
- g.m_cfg.appointment_types = [AppointmentType(routing_key=k) for k in ('a', 'b')]
- g.m_tasks = [Task(instance=MagicMock(), acceptable_routing_keys=[k]) for k in ('a', 'b')]
- g.redis_client.mget.return_value = [self.signal('a'), self.signal('b')]
- entered = {k: threading.Event() for k in ('a', 'b')}
- release = threading.Event()
- calls = []
- def job(task, result):
- key = result.apt_type.routing_key
- calls.append(key)
- entered[key].set()
- release.wait(3)
- g._execute_book_job = job
- try:
- g._poll_booking_signals()
- self.assertTrue(entered['a'].wait(2))
- self.assertTrue(entered['b'].wait(2))
- g._schedule_maintenance()
- for task in g.m_tasks:
- task.instance.keep_alive.assert_not_called()
- g._poll_booking_signals()
- self.assertCountEqual(calls, ['a', 'b'])
- self.assertEqual(g.redis_client.mget.call_count, 1)
- finally:
- release.set()
- def test_busy_browser_is_not_refreshed_by_maintenance(self):
- g = self.make()
- plugin = MagicMock()
- task = Task(instance=plugin, busy=True)
- g.m_tasks = [task]
- g._schedule_maintenance()
- plugin.keep_alive.assert_not_called()
- def test_cleanup_waits_for_inflight_booking(self):
- g = self.make()
- plugin = MagicMock()
- task = Task(instance=plugin, busy=True)
- g.m_tasks = [task]
- started = threading.Event()
- release = threading.Event()
- g._execute_book_job = lambda *_: (started.set(), release.wait(3))
- g._executor.submit(g._run_book_job, task, QueryResult())
- self.assertTrue(started.wait(2))
- stopper = threading.Thread(target=g.stop)
- stopper.start()
- try:
- self.assertTrue(g.m_stop_event.wait(2))
- plugin.cleanup.assert_not_called()
- finally:
- release.set()
- stopper.join(3)
- self.assertFalse(stopper.is_alive())
- plugin.cleanup.assert_called_once()
- def test_spawn_finishing_after_stop_is_cleaned_in_all_pool_modes(self):
- for cls, method in ((SentinelGCO, '_spawn_sentinel_worker'),
- (BuiltinBookerGCO, '_spawn_worker'),
- (OrderBookerGCO, '_spawn_worker')):
- with self.subTest(cls=cls):
- g = self.make(cls)
- self.cloud.get_vas_task_pop.return_value = {'id': 42, 'user_inputs': {'username': 'audit'}}
- plugin = MagicMock()
- plugin.create_session.side_effect = g.stop
- g.m_factory.create = MagicMock(return_value=plugin)
- getattr(g, method)(*(['a'] if cls is OrderBookerGCO else []))
- self.assertEqual(g.m_tasks, [])
- # Forced shutdown and late creator completion may both clean;
- # cleanup is idempotent and ownership must be fully released.
- plugin.cleanup.assert_called()
- self.assertEqual(g._resources._entries, {})
- def test_empty_group_does_not_poll_redis(self):
- g = self.make()
- g.m_cfg.appointment_types = [AppointmentType(routing_key='a')]
- g._poll_booking_signals()
- g.redis_client.mget.assert_not_called()
- def test_signal_reads_are_batched(self):
- g = self.make()
- keys = ['a', 'b', 'c']
- g.m_cfg.appointment_types = [AppointmentType(routing_key=k) for k in keys]
- g.m_tasks = [Task(instance=MagicMock(), acceptable_routing_keys=keys)]
- g.redis_client.mget.return_value = [None] * 3
- g._poll_booking_signals()
- g.redis_client.mget.assert_called_once_with(['vs:signal:' + k for k in keys])
- g.redis_client.get.assert_not_called()
- def test_available_signal_without_date_does_not_crash_or_book(self):
- g = self.make(OrderBookerGCO)
- result = QueryResult(availability_status=AvailabilityStatus.Available)
- self.assertFalse(g._is_date_of_interest(Task(instance=MagicMock()), result))
- def test_sentinel_slow_health_check_does_not_stall_other_instances(self):
- g = self.make(SentinelGCO)
- g.m_cfg.appointment_types = [AppointmentType(routing_key='a')]
- slow = MagicMock(config=PluginConfig())
- fast = MagicMock(config=PluginConfig())
- entered = threading.Event()
- release = threading.Event()
- queried = threading.Event()
- slow.health_check.side_effect = lambda: (entered.set(), release.wait(3), True)[-1]
- slow.query.return_value = QueryResult()
- fast.query.side_effect = lambda *_: (queried.set(), QueryResult())[-1]
- g.m_tasks = [Task(instance=slow), Task(instance=fast)]
- try:
- with patch('sentinel.time.time', return_value=1000):
- g._schedule_monitor_jobs()
- self.assertTrue(entered.wait(2))
- with patch('sentinel.time.time', return_value=1001):
- deadline = time.monotonic() + 1
- while g.m_tasks[1].busy and time.monotonic() < deadline:
- time.sleep(0.001)
- g._schedule_monitor_jobs()
- self.assertTrue(queried.wait(2))
- finally:
- release.set()
- def test_executor_rejects_overload_instead_of_queuing(self):
- executor = TaskExecutor(1, 'audit')
- release = threading.Event()
- try:
- self.assertTrue(executor.submit(release.wait, 2))
- self.assertFalse(executor.submit(lambda: None))
- finally:
- release.set()
- executor.shutdown()
- class SweeperTests(unittest.TestCase):
- def setUp(self):
- self.tmp = tempfile.TemporaryDirectory()
- self.addCleanup(self.tmp.cleanup)
- self.sweeper = OrphanTaskSweeper({}, logger=MagicMock())
- self.sweeper.redis_client = MagicMock()
- self.sweeper._outbox = MagicMock()
- self.sweeper._outbox.get.return_value = None
- self.sweeper.redis_client.get.return_value = None
- self.sweeper.redis_client.zscore.return_value = 0
- self.cloud = MagicMock()
- self.cloud.get_vas_task.return_value = {'status': 'processing'}
- patcher = patch('toolkit.cloud_api.CloudApi.Instance', return_value=self.cloud)
- patcher.start()
- self.addCleanup(patcher.stop)
- def test_scan_instead_of_keys(self):
- r = self.sweeper.redis_client
- r.scan_iter.return_value = [b'tracker']
- r.zrangebyscore.return_value = [b'42']
- with patch.object(self.sweeper, '_rescue_task') as rescue:
- self.sweeper._sweep()
- r.keys.assert_not_called()
- r.scan_iter.assert_called_once_with(match='vs:worker:tasks_tracker:*', count=100)
- rescue.assert_called_once_with('tracker', 42)
- def test_pending_success_is_never_requeued(self):
- self.sweeper._outbox.get.return_value = {'status': 'grabbed'}
- self.sweeper._rescue_task('tracker', 42)
- self.cloud.return_vas_task_to_queue.assert_not_called()
- def test_remote_success_marker_is_synchronized(self):
- self.sweeper.redis_client.get.return_value = '{"status": "grabbed"}'
- self.sweeper._rescue_task('tracker', 42)
- self.sweeper._outbox.put.assert_called_once_with(42, 'tracker', {'status': 'grabbed'})
- self.sweeper._outbox.flush.assert_called_once()
- self.cloud.return_vas_task_to_queue.assert_not_called()
- def test_lease_renewed_during_cloud_read_is_not_requeued(self):
- self.sweeper.redis_client.zscore.side_effect = [0, time.time() + 120]
- self.sweeper._rescue_task('tracker', 42)
- self.cloud.return_vas_task_to_queue.assert_not_called()
- def test_terminal_failed_order_is_not_requeued(self):
- self.cloud.get_vas_task.return_value = {'status': 'failed'}
- self.sweeper._rescue_task('tracker', 42)
- self.cloud.return_vas_task_to_queue.assert_not_called()
- self.sweeper.redis_client.zrem.assert_called_once_with('tracker', 42)
- class RedisTests(unittest.TestCase):
- def test_heartbeat_uses_one_zadd_for_entire_mapping(self):
- client = SafeRedisClient({})
- client._client = MagicMock()
- mapping = {str(i): 123 for i in range(100)}
- self.assertTrue(client.bulk_zadd('tracker', mapping))
- client._client.zadd.assert_called_once_with('tracker', mapping)
- client._client.pipeline.assert_not_called()
- if __name__ == '__main__':
- unittest.main()
|