"""Offline resource regressions, including a real SIGTERM and an owned child PID.""" import os import shutil import signal import subprocess import sys import tempfile import threading import time import unittest from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch from booker import BuiltinBookerGCO from core_types import BookerStandaloneConfig, GroupConfig from gco_wrapper import GCOWrapper, State from sentinel import SentinelGCO from standalone import StandaloneGCO from toolkit.task_executor import TaskExecutor from utils.plugin_utils import cleanup_browser_workspace from utils.resource_lifecycle import InstanceRegistry from utils.safe_redis_cli import SafeRedisClient class ResourceLifecycleTests(unittest.TestCase): def test_configuration_failure_removes_partially_initialized_workspace(self): with tempfile.TemporaryDirectory() as root: workspace = Path(root) / 'profile' workspace.mkdir() plugin = MagicMock() plugin.set_config.side_effect = ValueError('invalid config') plugin.cleanup.side_effect = lambda: shutil.rmtree(workspace) g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None) g.m_factory.create = MagicMock(return_value=plugin) with patch('standalone.ProxyManager.Instance') as manager: manager.return_value.get_next_proxy.return_value = {} with self.assertRaises(ValueError): g._init_instance() self.assertFalse(workspace.exists()) self.assertFalse(g._resources._entries) g.stop() def test_failed_cleanup_keeps_handle_and_retry_can_release_it(self): g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None) plugin = SimpleNamespace(page=MagicMock(), tunnel=None, root_workspace=None) attempts = [] def cleanup(): attempts.append(1) if len(attempts) == 1: # Even a plugin that drops its page prematurely can be recovered. plugin.page = None raise RuntimeError('cleanup failed') plugin.cleanup = cleanup page = plugin.page g.m_instance = plugin self.assertFalse(g._cleanup_instance()) self.assertIs(g.m_instance, plugin) self.assertTrue(g._resources._entries) g.stop() page.quit.assert_called() self.assertIsNone(g.m_instance) self.assertFalse(g._resources._entries) def test_browser_cleanup_failure_is_reported_and_tunnel_is_still_stopped(self): page = SimpleNamespace(process_id=None, quit=MagicMock(side_effect=RuntimeError('CDP failed'))) tunnel = MagicMock() with self.assertRaisesRegex(RuntimeError, 'retaining its handle'): cleanup_browser_workspace(page=page, tunnel=tunnel) self.assertEqual(page.quit.call_count, 2) tunnel.stop.assert_called_once() def test_force_fallback_terminates_only_owned_child(self): child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)']) try: page = SimpleNamespace(process_id=child.pid, quit=MagicMock(side_effect=RuntimeError('CDP failed'))) cleanup_browser_workspace(page=page) child.wait(timeout=2) self.assertIsNotNone(child.returncode) finally: if child.poll() is None: child.kill() child.wait() def test_partial_start_is_rolled_back_and_failed_wrapper_can_stop(self): class PartialStart(BuiltinBookerGCO): def _booking_trigger_loop(self): self.m_stop_event.wait() def _creator_loop(self): self.m_stop_event.wait() def _maintain_loop(self): self.m_stop_event.wait() def _heartbeat_loop(self): self.m_stop_event.wait() def _success_sync_loop(self): self.m_stop_event.wait() def _start_additional_threads(self): raise RuntimeError('partial startup') wrapper = GCOWrapper(PartialStart, GroupConfig(enable=True), {}) wrapper.load() with self.assertRaisesRegex(RuntimeError, 'partial startup'): wrapper.start() self.assertEqual(wrapper._state, State.FAILED) self.assertTrue(wrapper._gco.m_stop_event.is_set()) self.assertFalse(any(t.is_alive() for t in wrapper._gco._background_threads)) wrapper.stop() self.assertEqual(wrapper._state, State.STOPPED) def test_blocked_worker_gets_bounded_stop_and_resource_cancellation(self): g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.02, cleanup_timeout=0.1), {}, logger=lambda _: None) g.redis_client = MagicMock() release = threading.Event() entered = threading.Event() plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None, cleanup=lambda: release.set()) def run(): with g._resources.operation(plugin): entered.set() release.wait(2) g._executor.submit(run) self.assertTrue(entered.wait(1)) started = time.monotonic() try: try: g.stop() except RuntimeError: # An in-flight cleanup can complete immediately after the deadline. pass self.assertLess(time.monotonic() - started, 0.6) self.assertTrue(release.is_set()) finally: release.set() g._executor.shutdown(timeout=1) g.stop() g.redis_client.close.assert_called_once() def test_executor_does_not_keep_process_alive_for_a_blocked_call(self): executor = TaskExecutor(1, 'resource-test') release = threading.Event() entered = threading.Event() executor.submit(lambda: (entered.set(), release.wait(2))) self.assertTrue(entered.wait(1)) try: self.assertFalse(executor.shutdown(timeout=0.01)) self.assertTrue(all(t.daemon for t in executor._threads)) self.assertFalse(executor.submit(lambda: None)) finally: release.set() self.assertTrue(executor.shutdown(timeout=1)) def test_redis_is_not_closed_under_an_inflight_heartbeat(self): g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.01, cleanup_timeout=0.05), {}, logger=lambda _: None) g.redis_client = MagicMock() release = threading.Event() entered = threading.Event() thread = threading.Thread(target=lambda: (entered.set(), release.wait(2)), name='_heartbeat_loop', daemon=True) g._background_threads.append(thread) thread.start() self.assertTrue(entered.wait(1)) try: with self.assertRaises(RuntimeError): g.stop() g.redis_client.close.assert_not_called() finally: release.set() thread.join(1) g.stop() g.redis_client.close.assert_called_once() def test_failed_stop_is_retryable_without_overwriting_group(self): plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None, cleanup=MagicMock(side_effect=[RuntimeError('busy'), None])) g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None) g.redis_client = MagicMock() g.m_instance = plugin with self.assertRaises(RuntimeError): g.stop() self.assertIs(g.m_instance, plugin) g.redis_client.close.assert_not_called() g.stop() self.assertIsNone(g.m_instance) g.redis_client.close.assert_called_once() def test_stop_closes_redis_in_all_three_roles(self): for cls, cfg in ((StandaloneGCO, BookerStandaloneConfig()), (BuiltinBookerGCO, GroupConfig()), (SentinelGCO, GroupConfig())): with self.subTest(role=cls.__name__): g = cls(cfg, {}, logger=lambda _: None) raw = MagicMock() g.redis_client._client = raw g.stop() g.stop() raw.close.assert_called_once() raw.connection_pool.disconnect.assert_called_once() g.redis_client.get('after-close') raw.get.assert_not_called() def test_diagnostic_retention_does_not_defeat_explicit_stop(self): registry = InstanceRegistry(lambda _: None) plugin = SimpleNamespace(keep_browser_on_error=True, page=MagicMock(), tunnel=None, root_workspace=None, cleanup=MagicMock()) self.assertFalse(registry.cleanup(plugin)) self.assertTrue(registry.close()) self.assertFalse(plugin.keep_browser_on_error) def test_main_retains_failed_wrapper_for_cleanup_retry(self): import main as app class HalfStarted: def __init__(self, *args, **kwargs): pass def start(self): raise RuntimeError('startup failed') def stop(self): raise RuntimeError('cleanup pending') wrappers, classes = {}, {} config = {'identifier': 'retained', 'enable': True} with patch.object(app, 'get_gco_class', return_value=HalfStarted): app.sync_groups('sentinel', config, wrappers, classes, MagicMock()) self.assertIn('retained', wrappers) self.assertEqual(wrappers['retained']._state, State.FAILED) self.assertIsInstance(wrappers['retained']._gco, HalfStarted) def test_upgrade_listener_closes_pubsub_and_client_on_stop(self): from main import start_upgrade_listener stop = threading.Event() client = MagicMock() pubsub = client.pubsub.return_value pubsub.get_message.side_effect = lambda **_: (stop.set(), None)[1] with patch('redis.Redis', return_value=client): thread = start_upgrade_listener({}, 'test', MagicMock(), stop) thread.join(1) self.assertFalse(thread.is_alive()) pubsub.close.assert_called_once() client.close.assert_called_once() def test_stream_logger_closes_worker_and_pool(self): from utils.redis_stream_logger import RedisStreamLogger logger = RedisStreamLogger({}, 'test', 'test') raw = MagicMock() logger._redis_client = raw logger.close() self.assertFalse(logger._worker_thread.is_alive()) raw.close.assert_called_once() raw.connection_pool.disconnect.assert_called_once() @unittest.skipUnless(os.name == 'posix', 'POSIX signal integration') def test_sigterm_runs_main_finally_and_closes_connections(self): with tempfile.TemporaryDirectory() as root: ready = Path(root) / 'ready' done = Path(root) / 'done' script = ''' import sys from pathlib import Path from unittest.mock import MagicMock import main as app from core_types import GroupConfig, RoleStatus ready, done = map(Path, sys.argv[1:]) sys.argv = ['main.py', 'sentinel', '-c', 'unused.yaml'] logger = MagicMock() redis = MagicMock() stream = MagicMock() wrapper = MagicMock(m_cfg=GroupConfig()) wrapper.stop.side_effect = lambda: done.write_text('stopped') app.setup_app_logger = lambda *a, **k: logger app.SafeRedisClient = lambda *a, **k: redis app.RedisStreamLogger.initialize = lambda **k: stream app.start_upgrade_listener = lambda *a: None app.load_remote_or_cache = lambda **k: ([], '', False) app.collect_role_status = lambda *a: RoleStatus() def sync(role, config, wrappers, classes, logger): wrappers['test'] = wrapper ready.write_text('ready') app.sync_groups = sync app.main() assert redis.close.call_count == 1 assert stream.close.call_count == 1 ''' child = subprocess.Popen([sys.executable, '-c', script, str(ready), str(done)], cwd=str(Path(__file__).resolve().parent.parent), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: deadline = time.monotonic() + 3 while not ready.exists() and child.poll() is None and time.monotonic() < deadline: time.sleep(0.01) self.assertTrue(ready.exists()) child.send_signal(signal.SIGTERM) stdout, stderr = child.communicate(timeout=5) self.assertEqual(child.returncode, 0, stdout + stderr) self.assertTrue(done.exists()) finally: if child.poll() is None: child.kill() child.communicate() if __name__ == '__main__': unittest.main()