task_executor.py 2.7 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576
  1. """Bounded daemon workers with a finite shutdown wait.
  2. Python cannot safely terminate a blocked thread. Owners cancel their browser
  3. resources on timeout and retain a failed group until those calls unwind.
  4. """
  5. import logging
  6. import queue
  7. import threading
  8. from typing import Optional
  9. from utils.resource_lifecycle import join_threads
  10. class TaskExecutor:
  11. def __init__(self, max_workers: int, name: str):
  12. self._slots = threading.BoundedSemaphore(max_workers)
  13. self._queue = queue.Queue(maxsize=max_workers)
  14. self._lock = threading.Lock()
  15. self._closed = threading.Event()
  16. self._threads = []
  17. self._max_workers = max_workers
  18. self._name = name
  19. def _worker(self):
  20. while not self._closed.is_set() or not self._queue.empty():
  21. try:
  22. fn, args, kwargs = self._queue.get(timeout=0.05)
  23. except queue.Empty:
  24. continue
  25. try:
  26. fn(*args, **kwargs)
  27. except Exception:
  28. logging.getLogger(__name__).exception('Background job failed')
  29. finally:
  30. self._slots.release()
  31. self._queue.task_done()
  32. def submit(self, fn, *args, **kwargs):
  33. with self._lock:
  34. if self._closed.is_set() or not self._slots.acquire(blocking=False):
  35. return False
  36. try:
  37. if len(self._threads) < self._max_workers:
  38. thread = threading.Thread(target=self._worker, daemon=True,
  39. name=f'{self._name}-{len(self._threads)}')
  40. thread.start()
  41. self._threads.append(thread)
  42. self._queue.put_nowait((fn, args, kwargs))
  43. except Exception:
  44. self._slots.release()
  45. raise
  46. return True
  47. def shutdown(self, timeout=5):
  48. with self._lock:
  49. self._closed.set()
  50. return join_threads(self._threads, timeout)
  51. _background_executor: Optional[TaskExecutor] = None
  52. _bg_lock = threading.Lock()
  53. def get_background_executor(max_workers: int = 16) -> TaskExecutor:
  54. """Returns a shared daemon background TaskExecutor for lightweight async jobs."""
  55. global _background_executor
  56. if _background_executor is None:
  57. with _bg_lock:
  58. if _background_executor is None:
  59. _background_executor = TaskExecutor(max_workers=max_workers, name='Background')
  60. return _background_executor
  61. def submit_background(fn, *args, **kwargs) -> bool:
  62. """Convenience helper to submit a job to the shared background executor."""
  63. return get_background_executor().submit(fn, *args, **kwargs)