resource_lifecycle.py 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. """Keep ownership until cleanup succeeds, including partially initialized plugins."""
  2. import threading
  3. import time
  4. from functools import wraps
  5. from contextlib import contextmanager
  6. def init_lifecycle(owner):
  7. owner._lifecycle_lock = threading.RLock()
  8. owner._started = False
  9. owner._fully_stopped = False
  10. owner._resources = InstanceRegistry(owner._log)
  11. def start_once(method):
  12. @wraps(method)
  13. def start(owner):
  14. with owner._lifecycle_lock:
  15. if owner.m_stop_event.is_set():
  16. raise RuntimeError('A stopped group cannot be restarted; create a new group')
  17. if owner._started:
  18. return
  19. owner._started = True
  20. try:
  21. return method(owner)
  22. except Exception:
  23. try:
  24. owner.stop()
  25. except Exception as exc:
  26. owner._log(f'Startup rollback pending: {exc}')
  27. raise
  28. return start
  29. def serialized_stop(method):
  30. @wraps(method)
  31. def stop(owner):
  32. with owner._lifecycle_lock:
  33. if owner._fully_stopped:
  34. return
  35. method(owner)
  36. owner._fully_stopped = True
  37. return stop
  38. class InstanceRegistry:
  39. def __init__(self, logger):
  40. self._logger = logger
  41. self._lock = threading.RLock()
  42. self._entries = {}
  43. self._closing = False
  44. @property
  45. def empty(self):
  46. with self._lock:
  47. return not self._entries
  48. def track(self, instance):
  49. with self._lock:
  50. return self._entries.setdefault(id(instance), {
  51. 'instance': instance, 'active': 0, 'thread': None,
  52. 'cleaned': False, 'failed': False})
  53. @contextmanager
  54. def operation(self, instance):
  55. entry = self.track(instance)
  56. with self._lock:
  57. entry['active'] += 1
  58. closing = self._closing
  59. try:
  60. if closing:
  61. raise RuntimeError('Group is stopping')
  62. yield
  63. finally:
  64. with self._lock:
  65. entry['active'] -= 1
  66. closing = self._closing
  67. if entry['cleaned'] and not entry['active']:
  68. self._entries.pop(id(instance), None)
  69. if closing:
  70. self.cleanup(instance, force=True)
  71. def cleanup(self, instance, force=False, timeout=3):
  72. entry = self.track(instance)
  73. with self._lock:
  74. thread = entry['thread']
  75. force_thread = entry.get('force_thread')
  76. if (force and thread and thread.is_alive()
  77. and (not force_thread or not force_thread.is_alive())):
  78. def force_release():
  79. try:
  80. from utils.plugin_utils import cleanup_browser_workspace
  81. cleanup_browser_workspace(**entry['handles'], logger=self._logger)
  82. except Exception as exc:
  83. self._logger(f'Forced browser shutdown pending: {exc}')
  84. force_thread = threading.Thread(target=force_release, daemon=True,
  85. name='Instance-Force-Cleanup')
  86. entry['force_thread'] = force_thread
  87. force_thread.start()
  88. if not thread or not thread.is_alive():
  89. entry['cleaned'] = False
  90. # Keep handles even if a plugin clears its attributes on failure.
  91. previous = entry.get('handles', {})
  92. entry['handles'] = {name: getattr(instance, name, None) or previous.get(name)
  93. for name in ('page', 'tunnel', 'root_workspace')}
  94. def run():
  95. try:
  96. if force:
  97. instance.keep_browser_on_error = False
  98. from utils.plugin_utils import cleanup_browser_workspace
  99. cleanup_browser_workspace(**entry['handles'], logger=self._logger)
  100. for name in ('page', 'tunnel'):
  101. if getattr(instance, name, None) is entry['handles'][name]:
  102. setattr(instance, name, None)
  103. elif getattr(instance, 'keep_browser_on_error', False) is True:
  104. raise RuntimeError('Diagnostic browser retained until explicit stop')
  105. instance.cleanup()
  106. with self._lock:
  107. entry['cleaned'] = True
  108. entry['failed'] = False
  109. if not entry['active']:
  110. self._entries.pop(id(instance), None)
  111. except Exception as exc:
  112. with self._lock:
  113. entry['failed'] = True
  114. self._logger(f'Instance cleanup pending: {exc}')
  115. thread = threading.Thread(target=run, daemon=True, name='Instance-Cleanup')
  116. entry['thread'] = thread
  117. thread.start()
  118. if thread is not threading.current_thread():
  119. thread.join(max(0, timeout))
  120. with self._lock:
  121. if not entry['cleaned']:
  122. entry['failed'] = True
  123. return entry['cleaned']
  124. def retry_failed(self, timeout=0):
  125. with self._lock:
  126. failed = [e['instance'] for e in self._entries.values() if e['failed']]
  127. for instance in failed:
  128. self.cleanup(instance, timeout=timeout)
  129. with self._lock:
  130. return not any(e['failed'] for e in self._entries.values())
  131. def close(self, timeout=3):
  132. with self._lock:
  133. self._closing = True
  134. entries = list(self._entries.values())
  135. deadline = time.monotonic() + timeout
  136. for entry in entries:
  137. self.cleanup(entry['instance'], force=True, timeout=0)
  138. for entry in entries:
  139. if entry['thread']:
  140. entry['thread'].join(max(0, deadline - time.monotonic()))
  141. if entry.get('force_thread'):
  142. entry['force_thread'].join(max(0, deadline - time.monotonic()))
  143. with self._lock:
  144. return not self._entries
  145. def join_threads(threads, timeout):
  146. deadline = time.monotonic() + max(0, timeout)
  147. for thread in threads:
  148. if thread is not threading.current_thread() and thread.ident is not None:
  149. thread.join(max(0, deadline - time.monotonic()))
  150. return not any(t.is_alive() and t is not threading.current_thread() for t in threads)