task_executor.py 1.9 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556
  1. """Bounded daemon workers with a finite shutdown wait.
  2. Python cannot safely terminate a blocked thread. Owners cancel their browser
  3. resources on timeout and retain a failed group until those calls unwind.
  4. """
  5. import logging
  6. import queue
  7. import threading
  8. from utils.resource_lifecycle import join_threads
  9. class TaskExecutor:
  10. def __init__(self, max_workers, name):
  11. self._slots = threading.BoundedSemaphore(max_workers)
  12. self._queue = queue.Queue(maxsize=max_workers)
  13. self._lock = threading.Lock()
  14. self._closed = threading.Event()
  15. self._threads = []
  16. self._max_workers = max_workers
  17. self._name = name
  18. def _worker(self):
  19. while not self._closed.is_set() or not self._queue.empty():
  20. try:
  21. fn, args = self._queue.get(timeout=0.05)
  22. except queue.Empty:
  23. continue
  24. try:
  25. fn(*args)
  26. except Exception:
  27. logging.getLogger(__name__).exception('Background browser job failed')
  28. finally:
  29. self._slots.release()
  30. self._queue.task_done()
  31. def submit(self, fn, *args):
  32. with self._lock:
  33. if self._closed.is_set() or not self._slots.acquire(blocking=False):
  34. return False
  35. try:
  36. if len(self._threads) < self._max_workers:
  37. thread = threading.Thread(target=self._worker, daemon=True,
  38. name=f'{self._name}-{len(self._threads)}')
  39. thread.start()
  40. self._threads.append(thread)
  41. self._queue.put_nowait((fn, args))
  42. except Exception:
  43. self._slots.release()
  44. raise
  45. return True
  46. def shutdown(self, timeout=5):
  47. with self._lock:
  48. self._closed.set()
  49. return join_threads(self._threads, timeout)