| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168 |
- """Keep ownership until cleanup succeeds, including partially initialized plugins."""
- import threading
- import time
- from functools import wraps
- from contextlib import contextmanager
- def init_lifecycle(owner):
- owner._lifecycle_lock = threading.RLock()
- owner._started = False
- owner._fully_stopped = False
- owner._resources = InstanceRegistry(owner._log)
- def start_once(method):
- @wraps(method)
- def start(owner):
- with owner._lifecycle_lock:
- if owner.m_stop_event.is_set():
- raise RuntimeError('A stopped group cannot be restarted; create a new group')
- if owner._started:
- return
- owner._started = True
- try:
- return method(owner)
- except Exception:
- try:
- owner.stop()
- except Exception as exc:
- owner._log(f'Startup rollback pending: {exc}')
- raise
- return start
- def serialized_stop(method):
- @wraps(method)
- def stop(owner):
- with owner._lifecycle_lock:
- if owner._fully_stopped:
- return
- method(owner)
- owner._fully_stopped = True
- return stop
- class InstanceRegistry:
- def __init__(self, logger):
- self._logger = logger
- self._lock = threading.RLock()
- self._entries = {}
- self._closing = False
- @property
- def empty(self):
- with self._lock:
- return not self._entries
- def track(self, instance):
- with self._lock:
- return self._entries.setdefault(id(instance), {
- 'instance': instance, 'active': 0, 'thread': None,
- 'cleaned': False, 'failed': False})
- @contextmanager
- def operation(self, instance):
- entry = self.track(instance)
- with self._lock:
- entry['active'] += 1
- closing = self._closing
- try:
- if closing:
- raise RuntimeError('Group is stopping')
- yield
- finally:
- with self._lock:
- entry['active'] -= 1
- closing = self._closing
- if entry['cleaned'] and not entry['active']:
- self._entries.pop(id(instance), None)
- if closing:
- self.cleanup(instance, force=True)
- def cleanup(self, instance, force=False, timeout=3):
- entry = self.track(instance)
- with self._lock:
- thread = entry['thread']
- force_thread = entry.get('force_thread')
- if (force and thread and thread.is_alive()
- and (not force_thread or not force_thread.is_alive())):
- def force_release():
- try:
- from utils.plugin_utils import cleanup_browser_workspace
- cleanup_browser_workspace(**entry['handles'], logger=self._logger)
- except Exception as exc:
- self._logger(f'Forced browser shutdown pending: {exc}')
- force_thread = threading.Thread(target=force_release, daemon=True,
- name='Instance-Force-Cleanup')
- entry['force_thread'] = force_thread
- force_thread.start()
- if not thread or not thread.is_alive():
- entry['cleaned'] = False
- # Keep handles even if a plugin clears its attributes on failure.
- previous = entry.get('handles', {})
- entry['handles'] = {name: getattr(instance, name, None) or previous.get(name)
- for name in ('page', 'tunnel', 'root_workspace')}
- def run():
- try:
- if force:
- instance.keep_browser_on_error = False
- from utils.plugin_utils import cleanup_browser_workspace
- cleanup_browser_workspace(**entry['handles'], logger=self._logger)
- for name in ('page', 'tunnel'):
- if getattr(instance, name, None) is entry['handles'][name]:
- setattr(instance, name, None)
- elif getattr(instance, 'keep_browser_on_error', False) is True:
- raise RuntimeError('Diagnostic browser retained until explicit stop')
- instance.cleanup()
- with self._lock:
- entry['cleaned'] = True
- entry['failed'] = False
- if not entry['active']:
- self._entries.pop(id(instance), None)
- except Exception as exc:
- with self._lock:
- entry['failed'] = True
- self._logger(f'Instance cleanup pending: {exc}')
- thread = threading.Thread(target=run, daemon=True, name='Instance-Cleanup')
- entry['thread'] = thread
- thread.start()
- if thread is not threading.current_thread():
- thread.join(max(0, timeout))
- with self._lock:
- if not entry['cleaned']:
- entry['failed'] = True
- return entry['cleaned']
- def retry_failed(self, timeout=0):
- with self._lock:
- failed = [e['instance'] for e in self._entries.values() if e['failed']]
- for instance in failed:
- self.cleanup(instance, timeout=timeout)
- with self._lock:
- return not any(e['failed'] for e in self._entries.values())
- def close(self, timeout=3):
- with self._lock:
- self._closing = True
- entries = list(self._entries.values())
- deadline = time.monotonic() + timeout
- for entry in entries:
- self.cleanup(entry['instance'], force=True, timeout=0)
- for entry in entries:
- if entry['thread']:
- entry['thread'].join(max(0, deadline - time.monotonic()))
- if entry.get('force_thread'):
- entry['force_thread'].join(max(0, deadline - time.monotonic()))
- with self._lock:
- return not self._entries
- def join_threads(threads, timeout):
- deadline = time.monotonic() + max(0, timeout)
- for thread in threads:
- if thread is not threading.current_thread() and thread.ident is not None:
- thread.join(max(0, deadline - time.monotonic()))
- return not any(t.is_alive() and t is not threading.current_thread() for t in threads)
|