"""Keep ownership until cleanup succeeds, including partially initialized plugins.""" import threading import time from functools import wraps from contextlib import contextmanager def init_lifecycle(owner): owner._lifecycle_lock = threading.RLock() owner._started = False owner._fully_stopped = False owner._resources = InstanceRegistry(owner._log) def start_once(method): @wraps(method) def start(owner): with owner._lifecycle_lock: if owner.m_stop_event.is_set(): raise RuntimeError('A stopped group cannot be restarted; create a new group') if owner._started: return owner._started = True try: return method(owner) except Exception: try: owner.stop() except Exception as exc: owner._log(f'Startup rollback pending: {exc}') raise return start def serialized_stop(method): @wraps(method) def stop(owner): with owner._lifecycle_lock: if owner._fully_stopped: return method(owner) owner._fully_stopped = True return stop class InstanceRegistry: def __init__(self, logger): self._logger = logger self._lock = threading.RLock() self._entries = {} self._closing = False @property def empty(self): with self._lock: return not self._entries def track(self, instance): with self._lock: return self._entries.setdefault(id(instance), { 'instance': instance, 'active': 0, 'thread': None, 'cleaned': False, 'failed': False}) @contextmanager def operation(self, instance): entry = self.track(instance) with self._lock: entry['active'] += 1 closing = self._closing try: if closing: raise RuntimeError('Group is stopping') yield finally: with self._lock: entry['active'] -= 1 closing = self._closing if entry['cleaned'] and not entry['active']: self._entries.pop(id(instance), None) if closing: self.cleanup(instance, force=True) def cleanup(self, instance, force=False, timeout=3): entry = self.track(instance) with self._lock: thread = entry['thread'] force_thread = entry.get('force_thread') if (force and thread and thread.is_alive() and (not force_thread or not force_thread.is_alive())): def force_release(): try: from utils.plugin_utils import cleanup_browser_workspace cleanup_browser_workspace(**entry['handles'], logger=self._logger) except Exception as exc: self._logger(f'Forced browser shutdown pending: {exc}') force_thread = threading.Thread(target=force_release, daemon=True, name='Instance-Force-Cleanup') entry['force_thread'] = force_thread force_thread.start() if not thread or not thread.is_alive(): entry['cleaned'] = False # Keep handles even if a plugin clears its attributes on failure. previous = entry.get('handles', {}) entry['handles'] = {name: getattr(instance, name, None) or previous.get(name) for name in ('page', 'tunnel', 'root_workspace')} def run(): try: if force: instance.keep_browser_on_error = False from utils.plugin_utils import cleanup_browser_workspace cleanup_browser_workspace(**entry['handles'], logger=self._logger) for name in ('page', 'tunnel'): if getattr(instance, name, None) is entry['handles'][name]: setattr(instance, name, None) elif getattr(instance, 'keep_browser_on_error', False) is True: raise RuntimeError('Diagnostic browser retained until explicit stop') instance.cleanup() with self._lock: entry['cleaned'] = True entry['failed'] = False if not entry['active']: self._entries.pop(id(instance), None) except Exception as exc: with self._lock: entry['failed'] = True self._logger(f'Instance cleanup pending: {exc}') thread = threading.Thread(target=run, daemon=True, name='Instance-Cleanup') entry['thread'] = thread thread.start() if thread is not threading.current_thread(): thread.join(max(0, timeout)) with self._lock: if not entry['cleaned']: entry['failed'] = True return entry['cleaned'] def retry_failed(self, timeout=0): with self._lock: failed = [e['instance'] for e in self._entries.values() if e['failed']] for instance in failed: self.cleanup(instance, timeout=timeout) with self._lock: return not any(e['failed'] for e in self._entries.values()) def close(self, timeout=3): with self._lock: self._closing = True entries = list(self._entries.values()) deadline = time.monotonic() + timeout for entry in entries: self.cleanup(entry['instance'], force=True, timeout=0) for entry in entries: if entry['thread']: entry['thread'].join(max(0, deadline - time.monotonic())) if entry.get('force_thread'): entry['force_thread'].join(max(0, deadline - time.monotonic())) with self._lock: return not self._entries def join_threads(threads, timeout): deadline = time.monotonic() + max(0, timeout) for thread in threads: if thread is not threading.current_thread() and thread.ident is not None: thread.join(max(0, deadline - time.monotonic())) return not any(t.is_alive() and t is not threading.current_thread() for t in threads)