| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290 |
- """Offline resource regressions, including a real SIGTERM and an owned child PID."""
- import os
- import shutil
- import signal
- import subprocess
- import sys
- import tempfile
- import threading
- import time
- import unittest
- from pathlib import Path
- from types import SimpleNamespace
- from unittest.mock import MagicMock, patch
- from booker import BuiltinBookerGCO
- from core_types import BookerStandaloneConfig, GroupConfig
- from gco_wrapper import GCOWrapper, State
- from sentinel import SentinelGCO
- from standalone import StandaloneGCO
- from toolkit.task_executor import TaskExecutor
- from utils.plugin_utils import cleanup_browser_workspace
- from utils.resource_lifecycle import InstanceRegistry
- from utils.safe_redis_cli import SafeRedisClient
- class ResourceLifecycleTests(unittest.TestCase):
- def test_configuration_failure_removes_partially_initialized_workspace(self):
- with tempfile.TemporaryDirectory() as root:
- workspace = Path(root) / 'profile'
- workspace.mkdir()
- plugin = MagicMock()
- plugin.set_config.side_effect = ValueError('invalid config')
- plugin.cleanup.side_effect = lambda: shutil.rmtree(workspace)
- g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
- g.m_factory.create = MagicMock(return_value=plugin)
- with patch('standalone.ProxyManager.Instance') as manager:
- manager.return_value.get_next_proxy.return_value = {}
- with self.assertRaises(ValueError):
- g._init_instance()
- self.assertFalse(workspace.exists())
- self.assertFalse(g._resources._entries)
- g.stop()
- def test_failed_cleanup_keeps_handle_and_retry_can_release_it(self):
- g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
- plugin = SimpleNamespace(page=MagicMock(), tunnel=None, root_workspace=None)
- attempts = []
- def cleanup():
- attempts.append(1)
- if len(attempts) == 1:
- # Even a plugin that drops its page prematurely can be recovered.
- plugin.page = None
- raise RuntimeError('cleanup failed')
- plugin.cleanup = cleanup
- page = plugin.page
- g.m_instance = plugin
- self.assertFalse(g._cleanup_instance())
- self.assertIs(g.m_instance, plugin)
- self.assertTrue(g._resources._entries)
- g.stop()
- page.quit.assert_called()
- self.assertIsNone(g.m_instance)
- self.assertFalse(g._resources._entries)
- def test_browser_cleanup_failure_is_reported_and_tunnel_is_still_stopped(self):
- page = SimpleNamespace(process_id=None, quit=MagicMock(side_effect=RuntimeError('CDP failed')))
- tunnel = MagicMock()
- with self.assertRaisesRegex(RuntimeError, 'retaining its handle'):
- cleanup_browser_workspace(page=page, tunnel=tunnel)
- self.assertEqual(page.quit.call_count, 2)
- tunnel.stop.assert_called_once()
- def test_force_fallback_terminates_only_owned_child(self):
- child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])
- try:
- page = SimpleNamespace(process_id=child.pid,
- quit=MagicMock(side_effect=RuntimeError('CDP failed')))
- cleanup_browser_workspace(page=page)
- child.wait(timeout=2)
- self.assertIsNotNone(child.returncode)
- finally:
- if child.poll() is None:
- child.kill()
- child.wait()
- def test_partial_start_is_rolled_back_and_failed_wrapper_can_stop(self):
- class PartialStart(BuiltinBookerGCO):
- def _booking_trigger_loop(self): self.m_stop_event.wait()
- def _creator_loop(self): self.m_stop_event.wait()
- def _maintain_loop(self): self.m_stop_event.wait()
- def _heartbeat_loop(self): self.m_stop_event.wait()
- def _success_sync_loop(self): self.m_stop_event.wait()
- def _start_additional_threads(self): raise RuntimeError('partial startup')
- wrapper = GCOWrapper(PartialStart, GroupConfig(enable=True), {})
- wrapper.load()
- with self.assertRaisesRegex(RuntimeError, 'partial startup'):
- wrapper.start()
- self.assertEqual(wrapper._state, State.FAILED)
- self.assertTrue(wrapper._gco.m_stop_event.is_set())
- self.assertFalse(any(t.is_alive() for t in wrapper._gco._background_threads))
- wrapper.stop()
- self.assertEqual(wrapper._state, State.STOPPED)
- def test_blocked_worker_gets_bounded_stop_and_resource_cancellation(self):
- g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.02, cleanup_timeout=0.1), {}, logger=lambda _: None)
- g.redis_client = MagicMock()
- release = threading.Event()
- entered = threading.Event()
- plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None,
- cleanup=lambda: release.set())
- def run():
- with g._resources.operation(plugin):
- entered.set()
- release.wait(2)
- g._executor.submit(run)
- self.assertTrue(entered.wait(1))
- started = time.monotonic()
- try:
- try:
- g.stop()
- except RuntimeError:
- # An in-flight cleanup can complete immediately after the deadline.
- pass
- self.assertLess(time.monotonic() - started, 0.6)
- self.assertTrue(release.is_set())
- finally:
- release.set()
- g._executor.shutdown(timeout=1)
- g.stop()
- g.redis_client.close.assert_called_once()
- def test_executor_does_not_keep_process_alive_for_a_blocked_call(self):
- executor = TaskExecutor(1, 'resource-test')
- release = threading.Event()
- entered = threading.Event()
- executor.submit(lambda: (entered.set(), release.wait(2)))
- self.assertTrue(entered.wait(1))
- try:
- self.assertFalse(executor.shutdown(timeout=0.01))
- self.assertTrue(all(t.daemon for t in executor._threads))
- self.assertFalse(executor.submit(lambda: None))
- finally:
- release.set()
- self.assertTrue(executor.shutdown(timeout=1))
- def test_redis_is_not_closed_under_an_inflight_heartbeat(self):
- g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.01, cleanup_timeout=0.05), {}, logger=lambda _: None)
- g.redis_client = MagicMock()
- release = threading.Event()
- entered = threading.Event()
- thread = threading.Thread(target=lambda: (entered.set(), release.wait(2)),
- name='_heartbeat_loop', daemon=True)
- g._background_threads.append(thread)
- thread.start()
- self.assertTrue(entered.wait(1))
- try:
- with self.assertRaises(RuntimeError):
- g.stop()
- g.redis_client.close.assert_not_called()
- finally:
- release.set()
- thread.join(1)
- g.stop()
- g.redis_client.close.assert_called_once()
- def test_failed_stop_is_retryable_without_overwriting_group(self):
- plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None,
- cleanup=MagicMock(side_effect=[RuntimeError('busy'), None]))
- g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
- g.redis_client = MagicMock()
- g.m_instance = plugin
- with self.assertRaises(RuntimeError):
- g.stop()
- self.assertIs(g.m_instance, plugin)
- g.redis_client.close.assert_not_called()
- g.stop()
- self.assertIsNone(g.m_instance)
- g.redis_client.close.assert_called_once()
- def test_stop_closes_redis_in_all_three_roles(self):
- for cls, cfg in ((StandaloneGCO, BookerStandaloneConfig()),
- (BuiltinBookerGCO, GroupConfig()), (SentinelGCO, GroupConfig())):
- with self.subTest(role=cls.__name__):
- g = cls(cfg, {}, logger=lambda _: None)
- raw = MagicMock()
- g.redis_client._client = raw
- g.stop()
- g.stop()
- raw.close.assert_called_once()
- raw.connection_pool.disconnect.assert_called_once()
- g.redis_client.get('after-close')
- raw.get.assert_not_called()
- def test_diagnostic_retention_does_not_defeat_explicit_stop(self):
- registry = InstanceRegistry(lambda _: None)
- plugin = SimpleNamespace(keep_browser_on_error=True, page=MagicMock(),
- tunnel=None, root_workspace=None, cleanup=MagicMock())
- self.assertFalse(registry.cleanup(plugin))
- self.assertTrue(registry.close())
- self.assertFalse(plugin.keep_browser_on_error)
- def test_main_retains_failed_wrapper_for_cleanup_retry(self):
- import main as app
- class HalfStarted:
- def __init__(self, *args, **kwargs): pass
- def start(self): raise RuntimeError('startup failed')
- def stop(self): raise RuntimeError('cleanup pending')
- wrappers, classes = {}, {}
- config = {'identifier': 'retained', 'enable': True}
- with patch.object(app, 'get_gco_class', return_value=HalfStarted):
- app.sync_groups('sentinel', config, wrappers, classes, MagicMock())
- self.assertIn('retained', wrappers)
- self.assertEqual(wrappers['retained']._state, State.FAILED)
- self.assertIsInstance(wrappers['retained']._gco, HalfStarted)
- def test_upgrade_listener_closes_pubsub_and_client_on_stop(self):
- from main import start_upgrade_listener
- stop = threading.Event()
- client = MagicMock()
- pubsub = client.pubsub.return_value
- pubsub.get_message.side_effect = lambda **_: (stop.set(), None)[1]
- with patch('redis.Redis', return_value=client):
- thread = start_upgrade_listener({}, 'test', MagicMock(), stop)
- thread.join(1)
- self.assertFalse(thread.is_alive())
- pubsub.close.assert_called_once()
- client.close.assert_called_once()
- def test_stream_logger_closes_worker_and_pool(self):
- from utils.redis_stream_logger import RedisStreamLogger
- logger = RedisStreamLogger({}, 'test', 'test')
- raw = MagicMock()
- logger._redis_client = raw
- logger.close()
- self.assertFalse(logger._worker_thread.is_alive())
- raw.close.assert_called_once()
- raw.connection_pool.disconnect.assert_called_once()
- @unittest.skipUnless(os.name == 'posix', 'POSIX signal integration')
- def test_sigterm_runs_main_finally_and_closes_connections(self):
- with tempfile.TemporaryDirectory() as root:
- ready = Path(root) / 'ready'
- done = Path(root) / 'done'
- script = '''
- import sys
- from pathlib import Path
- from unittest.mock import MagicMock
- import main as app
- from core_types import GroupConfig, RoleStatus
- ready, done = map(Path, sys.argv[1:])
- sys.argv = ['main.py', 'sentinel', '-c', 'unused.yaml']
- logger = MagicMock()
- redis = MagicMock()
- stream = MagicMock()
- wrapper = MagicMock(m_cfg=GroupConfig())
- wrapper.stop.side_effect = lambda: done.write_text('stopped')
- app.setup_app_logger = lambda *a, **k: logger
- app.SafeRedisClient = lambda *a, **k: redis
- app.RedisStreamLogger.initialize = lambda **k: stream
- app.start_upgrade_listener = lambda *a: None
- app.load_remote_or_cache = lambda **k: ([], '', False)
- app.collect_role_status = lambda *a: RoleStatus()
- def sync(role, config, wrappers, classes, logger):
- wrappers['test'] = wrapper
- ready.write_text('ready')
- app.sync_groups = sync
- app.main()
- assert redis.close.call_count == 1
- assert stream.close.call_count == 1
- '''
- child = subprocess.Popen([sys.executable, '-c', script, str(ready), str(done)],
- cwd=str(Path(__file__).resolve().parent.parent),
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
- try:
- deadline = time.monotonic() + 3
- while not ready.exists() and child.poll() is None and time.monotonic() < deadline:
- time.sleep(0.01)
- self.assertTrue(ready.exists())
- child.send_signal(signal.SIGTERM)
- stdout, stderr = child.communicate(timeout=5)
- self.assertEqual(child.returncode, 0, stdout + stderr)
- self.assertTrue(done.exists())
- finally:
- if child.poll() is None:
- child.kill()
- child.communicate()
- if __name__ == '__main__':
- unittest.main()
|