|
|
@@ -1,4 +1,5 @@
|
|
|
import os
|
|
|
+from utils.resource_lifecycle import init_lifecycle, start_once, serialized_stop, join_threads
|
|
|
import time
|
|
|
import json
|
|
|
import threading
|
|
|
@@ -16,7 +17,8 @@ from core_types import (
|
|
|
from plugin_factory import PluginFactory
|
|
|
from toolkit.cloud_api import CloudApi
|
|
|
from toolkit.proxy_manager import ProxyManager
|
|
|
-from toolkit.thread_pool import ThreadPool
|
|
|
+from toolkit.task_executor import TaskExecutor
|
|
|
+from utils.booking_outbox import BookingOutbox
|
|
|
from toolkit.notifier import notify_book_result
|
|
|
from utils.safe_redis_cli import SafeRedisClient
|
|
|
|
|
|
@@ -35,9 +37,16 @@ class BaseBookerGCO:
|
|
|
self.m_lock = threading.RLock()
|
|
|
self.m_stop_event = threading.Event()
|
|
|
self.redis_client = SafeRedisClient(redis_conf, self.m_logger)
|
|
|
+ init_lifecycle(self)
|
|
|
self.m_tracker_key = f"vs:worker:tasks_tracker:{self.m_cfg.identifier}"
|
|
|
self.m_last_group_query_time = 0.0
|
|
|
self.m_spawn_cooldown_until = 0.0
|
|
|
+ self._executor = TaskExecutor(cfg.booker.max_concurrency, "Booking")
|
|
|
+ self._leased_orders = set()
|
|
|
+ self._pending_results = {}
|
|
|
+ self._outbox = BookingOutbox()
|
|
|
+ self._background_threads = []
|
|
|
+ self._heartbeat_stop_event = threading.Event()
|
|
|
|
|
|
def _log(self, message):
|
|
|
prefix = f"[-] [{self.TAG}] [{self.m_cfg.identifier}]"
|
|
|
@@ -72,6 +81,7 @@ class BaseBookerGCO:
|
|
|
updated_at=utc_now_str(),
|
|
|
)
|
|
|
|
|
|
+ @start_once
|
|
|
def start(self):
|
|
|
if not self.m_cfg.enable:
|
|
|
return
|
|
|
@@ -81,19 +91,42 @@ class BaseBookerGCO:
|
|
|
plugin_path = os.path.join(self.m_cfg.plugin_config.lib_path, self.m_cfg.plugin_config.plugin_bin)
|
|
|
self.m_factory.register_plugin(plugin_name, plugin_path, class_name)
|
|
|
|
|
|
- threading.Thread(target=self._booking_trigger_loop, daemon=True).start()
|
|
|
- threading.Thread(target=self._creator_loop, daemon=True).start()
|
|
|
- threading.Thread(target=self._maintain_loop, daemon=True).start()
|
|
|
+ for target in (self._booking_trigger_loop, self._creator_loop,
|
|
|
+ self._maintain_loop, self._heartbeat_loop, self._success_sync_loop):
|
|
|
+ thread = threading.Thread(target=target, daemon=True, name=target.__name__)
|
|
|
+ self._background_threads.append(thread)
|
|
|
+ thread.start()
|
|
|
self._start_additional_threads()
|
|
|
|
|
|
def _start_additional_threads(self):
|
|
|
"""Hook for subclasses to start additional background threads."""
|
|
|
pass
|
|
|
|
|
|
+ @serialized_stop
|
|
|
def stop(self):
|
|
|
self._log("Stopping Booker...")
|
|
|
self.m_stop_event.set()
|
|
|
- self._cleanup_all_tasks("booker stop")
|
|
|
+ deadline = time.monotonic() + self.m_cfg.shutdown_timeout
|
|
|
+ self._executor.shutdown(timeout=max(0, deadline - time.monotonic()))
|
|
|
+ workers = [t for t in self._background_threads if t.name != '_heartbeat_loop']
|
|
|
+ join_threads(workers, max(0, deadline - time.monotonic()))
|
|
|
+ with self.m_lock:
|
|
|
+ for task in self.m_tasks:
|
|
|
+ task.removed = True
|
|
|
+ self._resources.track(task.instance)
|
|
|
+ self._resources.close(self.m_cfg.cleanup_timeout)
|
|
|
+ threads_done = self._executor.shutdown(timeout=0.1) and join_threads(workers, 0.1)
|
|
|
+ if threads_done and not self._pending_results:
|
|
|
+ self._heartbeat_stop_event.set()
|
|
|
+ threads_done = join_threads(
|
|
|
+ [t for t in self._background_threads if t.name == '_heartbeat_loop'], 0.2)
|
|
|
+ if not self._resources.empty or not threads_done or self._pending_results:
|
|
|
+ raise RuntimeError('Booker shutdown incomplete; resources retained for retry')
|
|
|
+ with self.m_lock:
|
|
|
+ self.m_tasks.clear()
|
|
|
+ self._on_all_tasks_cleaned()
|
|
|
+ self._leased_orders.clear()
|
|
|
+ self.redis_client.close()
|
|
|
|
|
|
def update_config(self, new_cfg: GroupConfig):
|
|
|
"""Dynamically updates the configuration."""
|
|
|
@@ -108,24 +141,27 @@ class BaseBookerGCO:
|
|
|
self.m_cfg = new_cfg
|
|
|
|
|
|
def _cleanup_instance(self, instance, reason: str = "", task_ref=None):
|
|
|
- try:
|
|
|
- if instance:
|
|
|
- ref_str = f" for task={task_ref}" if task_ref else ""
|
|
|
- self._log(f"🧹 Cleaned up instance{ref_str}. Reason: {reason}")
|
|
|
- instance.cleanup()
|
|
|
- except Exception as e:
|
|
|
- self._log(f"Cleanup failed for instance. Reason: {reason}. Error: {e}")
|
|
|
+ if instance is None:
|
|
|
+ return True
|
|
|
+ cleaned = self._resources.cleanup(instance, timeout=self.m_cfg.cleanup_timeout)
|
|
|
+ if not cleaned:
|
|
|
+ self._log(f"Cleanup retained for retry ({reason})")
|
|
|
+ return cleaned
|
|
|
|
|
|
def _cleanup_task(self, task: Task, reason: str = ""):
|
|
|
if task and task.instance:
|
|
|
- self._cleanup_instance(task.instance, reason, task.task_ref)
|
|
|
+ with task._operation_lock:
|
|
|
+ self._cleanup_instance(task.instance, reason, task.task_ref)
|
|
|
|
|
|
def _remove_task(self, task: Task, reason: str = "", cleanup: bool = True):
|
|
|
removed = False
|
|
|
with self.m_lock:
|
|
|
if task in self.m_tasks:
|
|
|
+ task.removed = True
|
|
|
self.m_tasks.remove(task)
|
|
|
removed = True
|
|
|
+ if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
|
|
|
+ self._leased_orders.discard(str(task.task_ref))
|
|
|
self._on_task_removed(task)
|
|
|
|
|
|
if cleanup and removed:
|
|
|
@@ -140,6 +176,10 @@ class BaseBookerGCO:
|
|
|
with self.m_lock:
|
|
|
tasks = list(self.m_tasks)
|
|
|
self.m_tasks.clear()
|
|
|
+ for task in tasks:
|
|
|
+ task.removed = True
|
|
|
+ if task.task_ref is not None and str(task.task_ref) not in self._pending_results:
|
|
|
+ self._leased_orders.discard(str(task.task_ref))
|
|
|
self._on_all_tasks_cleaned()
|
|
|
for task in tasks:
|
|
|
self._cleanup_task(task, reason)
|
|
|
@@ -226,11 +266,8 @@ class BaseBookerGCO:
|
|
|
self._log(f"📋 VAS task [{effective_task_id}] marked as 'failed' due to account error: {biz_err.message}")
|
|
|
except Exception as update_err:
|
|
|
self._log(f"Failed to update VAS task status: {update_err}")
|
|
|
-
|
|
|
- try:
|
|
|
+ else:
|
|
|
self.redis_client.zrem(self.m_tracker_key, effective_task_id)
|
|
|
- except Exception as zrem_err:
|
|
|
- self._log(f"Failed to remove task from tracker: {zrem_err}")
|
|
|
|
|
|
# If pool account, permanently disable it in cloud
|
|
|
if acc and acc.id > 0 and acc.username not in ("", "Guest"):
|
|
|
@@ -338,93 +375,143 @@ class BaseBookerGCO:
|
|
|
else:
|
|
|
return current_utc_time >= start_time or current_utc_time <= end_time
|
|
|
|
|
|
- def _maintain_loop(self):
|
|
|
- self._log("Maintain loop started.")
|
|
|
- while not self.m_stop_event.is_set():
|
|
|
- try:
|
|
|
- time.sleep(1.0)
|
|
|
- with self.m_lock:
|
|
|
- tasks_to_check = list(self.m_tasks)
|
|
|
-
|
|
|
- if not tasks_to_check:
|
|
|
- continue
|
|
|
-
|
|
|
- dead_tasks = []
|
|
|
- healthy_tasks = []
|
|
|
- now = time.time()
|
|
|
- for t in tasks_to_check:
|
|
|
- if now >= t.next_remote_ping:
|
|
|
- t.instance.keep_alive()
|
|
|
- if t.instance.health_check():
|
|
|
- healthy_tasks.append(t)
|
|
|
- t.next_remote_ping = now + random.gauss(self.m_cfg.booker.keep_alive, 5)
|
|
|
- else:
|
|
|
- dead_tasks.append(t)
|
|
|
- else:
|
|
|
- healthy_tasks.append(t)
|
|
|
-
|
|
|
- self._on_maintain_ping(healthy_tasks, dead_tasks)
|
|
|
+ def _lease_order(self, task_id):
|
|
|
+ with self.m_lock:
|
|
|
+ self._leased_orders.add(str(task_id))
|
|
|
+ self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 120})
|
|
|
|
|
|
- if dead_tasks:
|
|
|
- for t in dead_tasks:
|
|
|
- self._remove_task(t, "unhealthy or keep-alive failed")
|
|
|
- except Exception as e:
|
|
|
- self._log(f'Maintain loop exception: {e}')
|
|
|
+ def _renew_leases(self):
|
|
|
+ with self.m_lock:
|
|
|
+ if self._leased_orders:
|
|
|
+ self.redis_client.bulk_zadd(self.m_tracker_key, {
|
|
|
+ tid: time.time() + 120 for tid in self._leased_orders})
|
|
|
|
|
|
- def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
|
|
|
- """Subclass hook for additional handling of healthy/dead tasks during maintenance."""
|
|
|
- pass
|
|
|
+ def _heartbeat_loop(self):
|
|
|
+ # No browser calls here: long OTP/HTTP waits must not stop order leases.
|
|
|
+ while not self._heartbeat_stop_event.wait(10):
|
|
|
+ self._renew_leases()
|
|
|
+
|
|
|
+ def _sync_successes(self):
|
|
|
+ try:
|
|
|
+ with self.m_lock:
|
|
|
+ pending = dict(self._pending_results)
|
|
|
+ for task_id, payload in pending.items():
|
|
|
+ saved = False
|
|
|
+ try:
|
|
|
+ self._outbox.put(task_id, self.m_tracker_key, payload)
|
|
|
+ saved = True
|
|
|
+ except Exception as exc:
|
|
|
+ self._log(f"Local success journal unavailable for task={task_id}: {exc}")
|
|
|
+ saved = bool(self.redis_client.set(self._outbox.marker(task_id), json.dumps(payload))) or saved
|
|
|
+ if saved:
|
|
|
+ with self.m_lock:
|
|
|
+ self._pending_results.pop(task_id, None)
|
|
|
+ self._leased_orders.discard(task_id)
|
|
|
+ self._outbox.flush(CloudApi.Instance(), self.redis_client._client,
|
|
|
+ self._log, tracker=self.m_tracker_key)
|
|
|
+ except Exception as exc:
|
|
|
+ self._log(f"Success outbox retry failed: {exc}")
|
|
|
+
|
|
|
+ def _success_sync_loop(self):
|
|
|
+ while not self.m_stop_event.is_set() or self._pending_results:
|
|
|
+ self._sync_successes()
|
|
|
+ if self.m_stop_event.is_set() and not self._pending_results:
|
|
|
+ break
|
|
|
+ if self.m_stop_event.is_set():
|
|
|
+ self._heartbeat_stop_event.wait(5)
|
|
|
+ else:
|
|
|
+ self.m_stop_event.wait(5)
|
|
|
+
|
|
|
+ def _maintain_task(self, task):
|
|
|
+ try:
|
|
|
+ with task._operation_lock, self._resources.operation(task.instance):
|
|
|
+ if task.removed or self.m_stop_event.is_set():
|
|
|
+ return
|
|
|
+ task.instance.keep_alive()
|
|
|
+ if task.instance.health_check():
|
|
|
+ task.next_remote_ping = time.time() + max(1, random.gauss(self.m_cfg.booker.keep_alive, 5))
|
|
|
+ else:
|
|
|
+ self._remove_task(task, "unhealthy or keep-alive failed")
|
|
|
+ except Exception as exc:
|
|
|
+ task.next_remote_ping = time.time() + 5
|
|
|
+ self._dispatch_error_policy("keep_alive", exc, task=task)
|
|
|
+ finally:
|
|
|
+ with self.m_lock:
|
|
|
+ task.busy = False
|
|
|
+
|
|
|
+ def _schedule_maintenance(self):
|
|
|
+ with self.m_lock:
|
|
|
+ if self.m_stop_event.is_set():
|
|
|
+ return
|
|
|
+ for task in self.m_tasks:
|
|
|
+ if task.busy or task.removed or time.time() < task.next_remote_ping:
|
|
|
+ continue
|
|
|
+ task.busy = True
|
|
|
+ if not self._executor.submit(self._maintain_task, task):
|
|
|
+ task.busy = False
|
|
|
+
|
|
|
+ def _maintain_loop(self):
|
|
|
+ while not self.m_stop_event.wait(1):
|
|
|
+ self._schedule_maintenance()
|
|
|
|
|
|
def _is_date_of_interest(self, task: Task, query_result: QueryResult) -> bool:
|
|
|
"""Checks if available dates in query_result match task preferences (defaults to True)."""
|
|
|
return True
|
|
|
|
|
|
+ def _run_book_job(self, task, query_result):
|
|
|
+ try:
|
|
|
+ with task._operation_lock, self._resources.operation(task.instance):
|
|
|
+ if not task.removed and not self.m_stop_event.is_set():
|
|
|
+ self._execute_book_job(task, query_result)
|
|
|
+ finally:
|
|
|
+ with self.m_lock:
|
|
|
+ task.next_run = max(task.next_run, time.time() + self.m_cfg.booker.booking_cooldown)
|
|
|
+ task.busy = False
|
|
|
+
|
|
|
+ def _poll_booking_signals(self):
|
|
|
+ now = time.time()
|
|
|
+ with self.m_lock:
|
|
|
+ ready = [t for t in self.m_tasks if not t.busy and not t.removed
|
|
|
+ and t.book_allowed and now >= t.next_run]
|
|
|
+ keys = {key for t in ready for key in t.acceptable_routing_keys}
|
|
|
+ apt_types = [a for a in self.m_cfg.appointment_types if a.routing_key in keys]
|
|
|
+ if not apt_types:
|
|
|
+ return
|
|
|
+ redis_keys = [self._get_redis_key(a.routing_key) for a in apt_types]
|
|
|
+ signals = self.redis_client.mget(redis_keys)
|
|
|
+ for apt_type, raw_data in zip(apt_types, signals):
|
|
|
+ if not raw_data:
|
|
|
+ continue
|
|
|
+ try:
|
|
|
+ data = json.loads(raw_data)
|
|
|
+ result = QueryResult.model_validate(data['query_result'])
|
|
|
+ result.apt_type = AppointmentType.model_validate(data['apt_type'])
|
|
|
+ if result.apt_type.routing_key != apt_type.routing_key:
|
|
|
+ raise ValueError('signal routing key mismatch')
|
|
|
+ except Exception as exc:
|
|
|
+ # A producer may have replaced this value since MGET. Do not
|
|
|
+ # delete a newer signal based on an old malformed payload.
|
|
|
+ self._log(f"Invalid signal for {apt_type.routing_key}: {exc}")
|
|
|
+ continue
|
|
|
+ with self.m_lock:
|
|
|
+ for task in ready:
|
|
|
+ if (task.busy or task.removed or self.m_stop_event.is_set()
|
|
|
+ or apt_type.routing_key not in task.acceptable_routing_keys
|
|
|
+ or not self._is_date_of_interest(task, result)):
|
|
|
+ continue
|
|
|
+ task.busy = True
|
|
|
+ if not self._executor.submit(self._run_book_job, task, result):
|
|
|
+ task.busy = False
|
|
|
+
|
|
|
def _booking_trigger_loop(self):
|
|
|
self._log("Trigger loop started.")
|
|
|
while not self.m_stop_event.is_set():
|
|
|
try:
|
|
|
- time.sleep(0.1)
|
|
|
- now = time.time()
|
|
|
- for apt_type in self.m_cfg.appointment_types:
|
|
|
- redis_key = self._get_redis_key(apt_type.routing_key)
|
|
|
- raw_data = self.redis_client.get(redis_key)
|
|
|
- if not raw_data:
|
|
|
- continue
|
|
|
- try:
|
|
|
- data = json.loads(raw_data)
|
|
|
- query_result = QueryResult.model_validate(data['query_result'])
|
|
|
- query_result.apt_type = AppointmentType.model_validate(data['apt_type'])
|
|
|
- except Exception as parse_err:
|
|
|
- self._log(f"Data parsing error for {redis_key}: {parse_err}. Deleting corrupted signal.")
|
|
|
- self.redis_client.delete(redis_key)
|
|
|
- continue
|
|
|
-
|
|
|
- matching_tasks = []
|
|
|
- with self.m_lock:
|
|
|
- for task in self.m_tasks:
|
|
|
- if now < task.next_run or not task.book_allowed:
|
|
|
- continue
|
|
|
- if apt_type.routing_key not in task.acceptable_routing_keys:
|
|
|
- continue
|
|
|
- if not self._is_date_of_interest(task, query_result):
|
|
|
- continue
|
|
|
-
|
|
|
- task.next_run = now + self.m_cfg.booker.booking_cooldown
|
|
|
- matching_tasks.append(task)
|
|
|
-
|
|
|
- if matching_tasks:
|
|
|
- threads = []
|
|
|
- for task in matching_tasks:
|
|
|
- self._log(f"🚀 Triggering BOOK for {apt_type.routing_key} | Order Ref: {task.task_ref}")
|
|
|
- t = threading.Thread(target=self._execute_book_job, args=(task, query_result))
|
|
|
- threads.append(t)
|
|
|
- t.start()
|
|
|
-
|
|
|
- for t in threads:
|
|
|
- t.join()
|
|
|
-
|
|
|
- except Exception as e:
|
|
|
- self._log(f"Booking trigger loop exception: {e}")
|
|
|
+ if self.m_cfg.enable and self._is_within_active_hours():
|
|
|
+ self._poll_booking_signals()
|
|
|
+ except Exception as exc:
|
|
|
+ self._log(f"Booking trigger loop exception: {exc}")
|
|
|
+ self.m_stop_event.wait(self.m_cfg.booker.signal_poll_interval)
|
|
|
|
|
|
def _execute_book_job(self, task: Task, query_result: QueryResult):
|
|
|
raise NotImplementedError
|
|
|
@@ -444,15 +531,16 @@ class BaseBookerGCO:
|
|
|
"payment_link": book_res.payment_link
|
|
|
}
|
|
|
|
|
|
- def _update_cloud_success():
|
|
|
- try:
|
|
|
- CloudApi.Instance().update_vas_task(str(task_id), {"status": "grabbed", "grabbed_history": grab_info})
|
|
|
- except Exception as e:
|
|
|
- self._log(f"Failed to update success state to cloud: {e}")
|
|
|
+ payload = {"status": "grabbed", "grabbed_history": grab_info}
|
|
|
+ # Commit locally before retiring the browser or dropping its lease.
|
|
|
+ with self.m_lock:
|
|
|
+ self._pending_results[str(task_id)] = payload
|
|
|
+ self._sync_successes()
|
|
|
+ try:
|
|
|
+ notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
|
|
|
+ except Exception as exc:
|
|
|
+ self._log(f"Booking succeeded, but notification could not be queued: {exc}")
|
|
|
|
|
|
- ThreadPool.getInstance().enqueue(_update_cloud_success)
|
|
|
- notify_book_result(book_res, order_id=str(order_id) if order_id else None, logger=self._log)
|
|
|
- self.redis_client.zrem(self.m_tracker_key, task_id)
|
|
|
|
|
|
|
|
|
class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
@@ -479,6 +567,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
self._spawn_worker()
|
|
|
|
|
|
def _spawn_worker(self):
|
|
|
+ if self.m_stop_event.is_set() or not self._resources.retry_failed():
|
|
|
+ return
|
|
|
instance = None
|
|
|
task = None
|
|
|
success = False
|
|
|
@@ -503,9 +593,11 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
|
|
|
|
|
|
instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
|
|
|
- instance.set_log(self.m_logger)
|
|
|
- instance.set_config(plg_cfg)
|
|
|
- instance.create_session()
|
|
|
+ self._resources.track(instance)
|
|
|
+ with self._resources.operation(instance):
|
|
|
+ instance.set_log(self.m_logger)
|
|
|
+ instance.set_config(plg_cfg)
|
|
|
+ instance.create_session()
|
|
|
|
|
|
all_keys = [apt.routing_key for apt in self.m_cfg.appointment_types]
|
|
|
task = Task(
|
|
|
@@ -518,6 +610,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
|
|
|
)
|
|
|
with self.m_lock:
|
|
|
+ if self.m_stop_event.is_set():
|
|
|
+ return
|
|
|
self.m_tasks.append(task)
|
|
|
|
|
|
success = True
|
|
|
@@ -554,7 +648,8 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
|
|
|
task_id = task_data.get('id')
|
|
|
order_id = task_data.get('order_id')
|
|
|
- self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 30.0})
|
|
|
+ self._lease_order(task_id)
|
|
|
+ book_res = None
|
|
|
|
|
|
try:
|
|
|
user_input = task_data.get('user_inputs', {})
|
|
|
@@ -572,7 +667,13 @@ class BuiltinBookerGCO(BaseBookerGCO):
|
|
|
else:
|
|
|
self._log(f"❌ BOOK FAILED for Order: {order_id}")
|
|
|
except Exception as e:
|
|
|
- self._dispatch_error_policy("book", e, task=task)
|
|
|
+ self._dispatch_error_policy("book", e, task=task, task_id=task_id)
|
|
|
+ finally:
|
|
|
+ with self.m_lock:
|
|
|
+ if str(task_id) not in self._pending_results:
|
|
|
+ self._leased_orders.discard(str(task_id))
|
|
|
+ if book_res is None or not book_res.success:
|
|
|
+ self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time()})
|
|
|
|
|
|
|
|
|
class OrderBookerGCO(BaseBookerGCO):
|
|
|
@@ -587,36 +688,28 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
def __init__(self, cfg: GroupConfig, redis_conf: Dict, logger: Callable[[str], None] = None):
|
|
|
super().__init__(cfg, redis_conf, logger)
|
|
|
self.m_task_data_cache: Dict[str, dict] = {}
|
|
|
- self.heartbeat_ttl = 2 * 60.0
|
|
|
|
|
|
def _start_additional_threads(self):
|
|
|
- threading.Thread(target=self._cache_refresh_loop, daemon=True).start()
|
|
|
+ thread = threading.Thread(target=self._cache_refresh_loop, daemon=True)
|
|
|
+ self._background_threads.append(thread)
|
|
|
+ thread.start()
|
|
|
|
|
|
def _on_task_removed(self, task: Task):
|
|
|
task_id = task.task_ref
|
|
|
- if task_id:
|
|
|
+ if task_id is not None:
|
|
|
self.m_task_data_cache.pop(str(task_id), None)
|
|
|
|
|
|
def _on_all_tasks_cleaned(self):
|
|
|
self.m_task_data_cache.clear()
|
|
|
|
|
|
- def _on_maintain_ping(self, healthy_tasks: List[Task], dead_tasks: List[Task]):
|
|
|
- if healthy_tasks:
|
|
|
- new_deadline = time.time() + self.heartbeat_ttl
|
|
|
- mapping = {str(t.task_ref): new_deadline for t in healthy_tasks}
|
|
|
- self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
|
|
|
-
|
|
|
- if dead_tasks:
|
|
|
- mapping = {str(t.task_ref): 0 for t in dead_tasks}
|
|
|
- self.redis_client.bulk_zadd(self.m_tracker_key, mapping)
|
|
|
-
|
|
|
def _cache_refresh_loop(self):
|
|
|
self._log("Cache refresh loop started.")
|
|
|
refresh_interval = 15 * 60
|
|
|
|
|
|
while not self.m_stop_event.is_set():
|
|
|
try:
|
|
|
- time.sleep(1)
|
|
|
+ if self.m_stop_event.wait(1):
|
|
|
+ break
|
|
|
with self.m_lock:
|
|
|
tasks_to_check = {
|
|
|
tid: data.get('_last_refresh', 0)
|
|
|
@@ -628,6 +721,8 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
|
|
|
now = time.time()
|
|
|
for tid, last_refresh in tasks_to_check.items():
|
|
|
+ if self.m_stop_event.is_set():
|
|
|
+ break
|
|
|
if now - last_refresh >= refresh_interval:
|
|
|
fresh_data = CloudApi.Instance().get_vas_task(tid)
|
|
|
if fresh_data:
|
|
|
@@ -636,7 +731,7 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
with self.m_lock:
|
|
|
if tid in self.m_task_data_cache:
|
|
|
self.m_task_data_cache[tid] = fresh_data
|
|
|
- time.sleep(0.5)
|
|
|
+ self.m_stop_event.wait(0.5)
|
|
|
except Exception as e:
|
|
|
self._log(f'Cache refresh loop exception: {e}')
|
|
|
|
|
|
@@ -651,6 +746,8 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
or '2100-01-01'
|
|
|
)
|
|
|
available_date = query_result.earliest_date
|
|
|
+ if available_date is None:
|
|
|
+ return False
|
|
|
dt = available_date.strftime("%Y-%m-%d")
|
|
|
return dt <= expected_end_date
|
|
|
|
|
|
@@ -661,7 +758,7 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
try:
|
|
|
with self.m_lock:
|
|
|
task_data = self.m_task_data_cache.get(str(task_id))
|
|
|
- if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled']:
|
|
|
+ if not task_data or task_data.get('status') in ['grabbed', 'pause', 'completed', 'cancelled', 'failed', 'success']:
|
|
|
self._log(f"Bound Task={task_id} is no longer valid or already processed. Removing instance.")
|
|
|
self._remove_task(task, "bound task no longer valid")
|
|
|
self.redis_client.zrem(self.m_tracker_key, task_id)
|
|
|
@@ -699,6 +796,8 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
self._spawn_worker(r_key)
|
|
|
|
|
|
def _spawn_worker(self, target_routing_key: str):
|
|
|
+ if self.m_stop_event.is_set() or not self._resources.retry_failed():
|
|
|
+ return
|
|
|
instance = None
|
|
|
task = None
|
|
|
success = False
|
|
|
@@ -720,7 +819,7 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
with self.m_lock:
|
|
|
self.m_task_data_cache[str(task_id)] = task_data
|
|
|
|
|
|
- self.redis_client.zadd(self.m_tracker_key, {str(task_id): time.time() + 8*60.0})
|
|
|
+ self._lease_order(task_id)
|
|
|
user_inputs = task_data.get('user_inputs', {})
|
|
|
|
|
|
plg_cfg = PluginConfig()
|
|
|
@@ -742,9 +841,11 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
plg_cfg.proxy = type(plg_cfg.proxy)(**proxy)
|
|
|
|
|
|
instance = self.m_factory.create(self.m_cfg.identifier, self.m_cfg.plugin_config.plugin_name)
|
|
|
- instance.set_log(self.m_logger)
|
|
|
- instance.set_config(plg_cfg)
|
|
|
- instance.create_session()
|
|
|
+ self._resources.track(instance)
|
|
|
+ with self._resources.operation(instance):
|
|
|
+ instance.set_log(self.m_logger)
|
|
|
+ instance.set_config(plg_cfg)
|
|
|
+ instance.create_session()
|
|
|
|
|
|
task = Task(
|
|
|
instance=instance,
|
|
|
@@ -756,6 +857,8 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
next_remote_ping=time.time() + random.gauss(self.m_cfg.booker.keep_alive, 5)
|
|
|
)
|
|
|
with self.m_lock:
|
|
|
+ if self.m_stop_event.is_set():
|
|
|
+ return
|
|
|
self.m_tasks.append(task)
|
|
|
|
|
|
success = True
|
|
|
@@ -773,10 +876,14 @@ class OrderBookerGCO(BaseBookerGCO):
|
|
|
finally:
|
|
|
if not success and instance:
|
|
|
self._cleanup_instance(instance, "spawn failed")
|
|
|
- if not success and task_id:
|
|
|
+ if not success and task_id is not None:
|
|
|
with self.m_lock:
|
|
|
self.m_task_data_cache.pop(str(task_id), None)
|
|
|
try:
|
|
|
- self.redis_client.zrem(self.m_tracker_key, task_id)
|
|
|
+ with self.m_lock:
|
|
|
+ self._leased_orders.discard(str(task_id))
|
|
|
+ # Retain a recovery deadline (including any 429 cooldown).
|
|
|
+ self.redis_client.zadd(self.m_tracker_key, {
|
|
|
+ str(task_id): max(time.time(), self.m_spawn_cooldown_until)})
|
|
|
except Exception:
|
|
|
- pass
|
|
|
+ pass
|