test_resource_lifecycle.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. """Offline resource regressions, including a real SIGTERM and an owned child PID."""
  2. import os
  3. import shutil
  4. import signal
  5. import subprocess
  6. import sys
  7. import tempfile
  8. import threading
  9. import time
  10. import unittest
  11. from pathlib import Path
  12. from types import SimpleNamespace
  13. from unittest.mock import MagicMock, patch
  14. from booker import BuiltinBookerGCO
  15. from core_types import BookerStandaloneConfig, GroupConfig
  16. from gco_wrapper import GCOWrapper, State
  17. from sentinel import SentinelGCO
  18. from standalone import StandaloneGCO
  19. from toolkit.task_executor import TaskExecutor
  20. from utils.plugin_utils import cleanup_browser_workspace
  21. from utils.resource_lifecycle import InstanceRegistry
  22. from utils.safe_redis_cli import SafeRedisClient
  23. class ResourceLifecycleTests(unittest.TestCase):
  24. def test_configuration_failure_removes_partially_initialized_workspace(self):
  25. with tempfile.TemporaryDirectory() as root:
  26. workspace = Path(root) / 'profile'
  27. workspace.mkdir()
  28. plugin = MagicMock()
  29. plugin.set_config.side_effect = ValueError('invalid config')
  30. plugin.cleanup.side_effect = lambda: shutil.rmtree(workspace)
  31. g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
  32. g.m_factory.create = MagicMock(return_value=plugin)
  33. with patch('standalone.ProxyManager.Instance') as manager:
  34. manager.return_value.get_next_proxy.return_value = {}
  35. with self.assertRaises(ValueError):
  36. g._init_instance()
  37. self.assertFalse(workspace.exists())
  38. self.assertFalse(g._resources._entries)
  39. g.stop()
  40. def test_failed_cleanup_keeps_handle_and_retry_can_release_it(self):
  41. g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
  42. plugin = SimpleNamespace(page=MagicMock(), tunnel=None, root_workspace=None)
  43. attempts = []
  44. def cleanup():
  45. attempts.append(1)
  46. if len(attempts) == 1:
  47. # Even a plugin that drops its page prematurely can be recovered.
  48. plugin.page = None
  49. raise RuntimeError('cleanup failed')
  50. plugin.cleanup = cleanup
  51. page = plugin.page
  52. g.m_instance = plugin
  53. self.assertFalse(g._cleanup_instance())
  54. self.assertIs(g.m_instance, plugin)
  55. self.assertTrue(g._resources._entries)
  56. g.stop()
  57. page.quit.assert_called()
  58. self.assertIsNone(g.m_instance)
  59. self.assertFalse(g._resources._entries)
  60. def test_browser_cleanup_failure_is_reported_and_tunnel_is_still_stopped(self):
  61. page = SimpleNamespace(process_id=None, quit=MagicMock(side_effect=RuntimeError('CDP failed')))
  62. tunnel = MagicMock()
  63. with self.assertRaisesRegex(RuntimeError, 'retaining its handle'):
  64. cleanup_browser_workspace(page=page, tunnel=tunnel)
  65. self.assertEqual(page.quit.call_count, 2)
  66. tunnel.stop.assert_called_once()
  67. def test_force_fallback_terminates_only_owned_child(self):
  68. child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])
  69. try:
  70. page = SimpleNamespace(process_id=child.pid,
  71. quit=MagicMock(side_effect=RuntimeError('CDP failed')))
  72. cleanup_browser_workspace(page=page)
  73. child.wait(timeout=2)
  74. self.assertIsNotNone(child.returncode)
  75. finally:
  76. if child.poll() is None:
  77. child.kill()
  78. child.wait()
  79. def test_partial_start_is_rolled_back_and_failed_wrapper_can_stop(self):
  80. class PartialStart(BuiltinBookerGCO):
  81. def _booking_trigger_loop(self): self.m_stop_event.wait()
  82. def _creator_loop(self): self.m_stop_event.wait()
  83. def _maintain_loop(self): self.m_stop_event.wait()
  84. def _heartbeat_loop(self): self.m_stop_event.wait()
  85. def _success_sync_loop(self): self.m_stop_event.wait()
  86. def _start_additional_threads(self): raise RuntimeError('partial startup')
  87. wrapper = GCOWrapper(PartialStart, GroupConfig(enable=True), {})
  88. wrapper.load()
  89. with self.assertRaisesRegex(RuntimeError, 'partial startup'):
  90. wrapper.start()
  91. self.assertEqual(wrapper._state, State.FAILED)
  92. self.assertTrue(wrapper._gco.m_stop_event.is_set())
  93. self.assertFalse(any(t.is_alive() for t in wrapper._gco._background_threads))
  94. wrapper.stop()
  95. self.assertEqual(wrapper._state, State.STOPPED)
  96. def test_blocked_worker_gets_bounded_stop_and_resource_cancellation(self):
  97. g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.02, cleanup_timeout=0.1), {}, logger=lambda _: None)
  98. g.redis_client = MagicMock()
  99. release = threading.Event()
  100. entered = threading.Event()
  101. plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None,
  102. cleanup=lambda: release.set())
  103. def run():
  104. with g._resources.operation(plugin):
  105. entered.set()
  106. release.wait(2)
  107. g._executor.submit(run)
  108. self.assertTrue(entered.wait(1))
  109. started = time.monotonic()
  110. try:
  111. try:
  112. g.stop()
  113. except RuntimeError:
  114. # An in-flight cleanup can complete immediately after the deadline.
  115. pass
  116. self.assertLess(time.monotonic() - started, 0.6)
  117. self.assertTrue(release.is_set())
  118. finally:
  119. release.set()
  120. g._executor.shutdown(timeout=1)
  121. g.stop()
  122. g.redis_client.close.assert_called_once()
  123. def test_executor_does_not_keep_process_alive_for_a_blocked_call(self):
  124. executor = TaskExecutor(1, 'resource-test')
  125. release = threading.Event()
  126. entered = threading.Event()
  127. executor.submit(lambda: (entered.set(), release.wait(2)))
  128. self.assertTrue(entered.wait(1))
  129. try:
  130. self.assertFalse(executor.shutdown(timeout=0.01))
  131. self.assertTrue(all(t.daemon for t in executor._threads))
  132. self.assertFalse(executor.submit(lambda: None))
  133. finally:
  134. release.set()
  135. self.assertTrue(executor.shutdown(timeout=1))
  136. def test_redis_is_not_closed_under_an_inflight_heartbeat(self):
  137. g = BuiltinBookerGCO(GroupConfig(shutdown_timeout=0.01, cleanup_timeout=0.05), {}, logger=lambda _: None)
  138. g.redis_client = MagicMock()
  139. release = threading.Event()
  140. entered = threading.Event()
  141. thread = threading.Thread(target=lambda: (entered.set(), release.wait(2)),
  142. name='_heartbeat_loop', daemon=True)
  143. g._background_threads.append(thread)
  144. thread.start()
  145. self.assertTrue(entered.wait(1))
  146. try:
  147. with self.assertRaises(RuntimeError):
  148. g.stop()
  149. g.redis_client.close.assert_not_called()
  150. finally:
  151. release.set()
  152. thread.join(1)
  153. g.stop()
  154. g.redis_client.close.assert_called_once()
  155. def test_failed_stop_is_retryable_without_overwriting_group(self):
  156. plugin = SimpleNamespace(page=None, tunnel=None, root_workspace=None,
  157. cleanup=MagicMock(side_effect=[RuntimeError('busy'), None]))
  158. g = StandaloneGCO(BookerStandaloneConfig(), {}, logger=lambda _: None)
  159. g.redis_client = MagicMock()
  160. g.m_instance = plugin
  161. with self.assertRaises(RuntimeError):
  162. g.stop()
  163. self.assertIs(g.m_instance, plugin)
  164. g.redis_client.close.assert_not_called()
  165. g.stop()
  166. self.assertIsNone(g.m_instance)
  167. g.redis_client.close.assert_called_once()
  168. def test_stop_closes_redis_in_all_three_roles(self):
  169. for cls, cfg in ((StandaloneGCO, BookerStandaloneConfig()),
  170. (BuiltinBookerGCO, GroupConfig()), (SentinelGCO, GroupConfig())):
  171. with self.subTest(role=cls.__name__):
  172. g = cls(cfg, {}, logger=lambda _: None)
  173. raw = MagicMock()
  174. g.redis_client._client = raw
  175. g.stop()
  176. g.stop()
  177. raw.close.assert_called_once()
  178. raw.connection_pool.disconnect.assert_called_once()
  179. g.redis_client.get('after-close')
  180. raw.get.assert_not_called()
  181. def test_diagnostic_retention_does_not_defeat_explicit_stop(self):
  182. registry = InstanceRegistry(lambda _: None)
  183. plugin = SimpleNamespace(keep_browser_on_error=True, page=MagicMock(),
  184. tunnel=None, root_workspace=None, cleanup=MagicMock())
  185. self.assertFalse(registry.cleanup(plugin))
  186. self.assertTrue(registry.close())
  187. self.assertFalse(plugin.keep_browser_on_error)
  188. def test_main_retains_failed_wrapper_for_cleanup_retry(self):
  189. import main as app
  190. class HalfStarted:
  191. def __init__(self, *args, **kwargs): pass
  192. def start(self): raise RuntimeError('startup failed')
  193. def stop(self): raise RuntimeError('cleanup pending')
  194. wrappers, classes = {}, {}
  195. config = {'identifier': 'retained', 'enable': True}
  196. with patch.object(app, 'get_gco_class', return_value=HalfStarted):
  197. app.sync_groups('sentinel', config, wrappers, classes, MagicMock())
  198. self.assertIn('retained', wrappers)
  199. self.assertEqual(wrappers['retained']._state, State.FAILED)
  200. self.assertIsInstance(wrappers['retained']._gco, HalfStarted)
  201. def test_upgrade_listener_closes_pubsub_and_client_on_stop(self):
  202. from main import start_upgrade_listener
  203. stop = threading.Event()
  204. client = MagicMock()
  205. pubsub = client.pubsub.return_value
  206. pubsub.get_message.side_effect = lambda **_: (stop.set(), None)[1]
  207. with patch('redis.Redis', return_value=client):
  208. thread = start_upgrade_listener({}, 'test', MagicMock(), stop)
  209. thread.join(1)
  210. self.assertFalse(thread.is_alive())
  211. pubsub.close.assert_called_once()
  212. client.close.assert_called_once()
  213. def test_stream_logger_closes_worker_and_pool(self):
  214. from utils.redis_stream_logger import RedisStreamLogger
  215. logger = RedisStreamLogger({}, 'test', 'test')
  216. raw = MagicMock()
  217. logger._redis_client = raw
  218. logger.close()
  219. self.assertFalse(logger._worker_thread.is_alive())
  220. raw.close.assert_called_once()
  221. raw.connection_pool.disconnect.assert_called_once()
  222. @unittest.skipUnless(os.name == 'posix', 'POSIX signal integration')
  223. def test_sigterm_runs_main_finally_and_closes_connections(self):
  224. with tempfile.TemporaryDirectory() as root:
  225. ready = Path(root) / 'ready'
  226. done = Path(root) / 'done'
  227. script = '''
  228. import sys
  229. from pathlib import Path
  230. from unittest.mock import MagicMock
  231. import main as app
  232. from core_types import GroupConfig, RoleStatus
  233. ready, done = map(Path, sys.argv[1:])
  234. sys.argv = ['main.py', 'sentinel', '-c', 'unused.yaml']
  235. logger = MagicMock()
  236. redis = MagicMock()
  237. stream = MagicMock()
  238. wrapper = MagicMock(m_cfg=GroupConfig())
  239. wrapper.stop.side_effect = lambda: done.write_text('stopped')
  240. app.setup_app_logger = lambda *a, **k: logger
  241. app.SafeRedisClient = lambda *a, **k: redis
  242. app.RedisStreamLogger.initialize = lambda **k: stream
  243. app.start_upgrade_listener = lambda *a: None
  244. app.load_remote_or_cache = lambda **k: ([], '', False)
  245. app.collect_role_status = lambda *a: RoleStatus()
  246. def sync(role, config, wrappers, classes, logger):
  247. wrappers['test'] = wrapper
  248. ready.write_text('ready')
  249. app.sync_groups = sync
  250. app.main()
  251. assert redis.close.call_count == 1
  252. assert stream.close.call_count == 1
  253. '''
  254. child = subprocess.Popen([sys.executable, '-c', script, str(ready), str(done)],
  255. cwd=str(Path(__file__).resolve().parent.parent),
  256. stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
  257. try:
  258. deadline = time.monotonic() + 3
  259. while not ready.exists() and child.poll() is None and time.monotonic() < deadline:
  260. time.sleep(0.01)
  261. self.assertTrue(ready.exists())
  262. child.send_signal(signal.SIGTERM)
  263. stdout, stderr = child.communicate(timeout=5)
  264. self.assertEqual(child.returncode, 0, stdout + stderr)
  265. self.assertTrue(done.exists())
  266. finally:
  267. if child.poll() is None:
  268. child.kill()
  269. child.communicate()
  270. if __name__ == '__main__':
  271. unittest.main()