"""Job queue and live-event plumbing. Redis is the transport. When it is not reachable the API still works: runs are executed inline as a background task and progress falls back to polling. That keeps local development possible without Redis, and keeps a Redis outage from taking the whole product down. """ from __future__ import annotations import asyncio import contextlib import json import logging import uuid from collections.abc import AsyncIterator from app.core.config import settings log = logging.getLogger("contiai.queue") _pool = None _pool_failed = False CANCEL_KEY = "contiai:cancel:{run_id}" EVENTS_KEY = "contiai:events:{run_id}" async def get_pool(): """ARQ pool, or None when Redis is unreachable.""" global _pool, _pool_failed if _pool is not None or _pool_failed: return _pool try: from arq import create_pool from arq.connections import RedisSettings _pool = await create_pool(RedisSettings.from_dsn(settings.redis_url)) return _pool except Exception as e: _pool_failed = True log.warning("Redis unavailable (%s); running jobs inline", e) return None def redis_available() -> bool: """Whether the queue is live, without attempting a connection. Read after `get_pool()` has run once at startup. The distinction matters for reporting: a fallback that works twenty-seven times out of twenty-eight reads as random flakiness unless the log states which mode the run was in. """ return _pool is not None async def close_pool() -> None: global _pool if _pool is not None: await _pool.close() _pool = None async def enqueue_run(run_id: uuid.UUID) -> None: pool = await get_pool() if pool is not None: await pool.enqueue_job("run_pipeline", str(run_id)) return # No worker available - execute in this process so the run still happens. from app.db.session import SessionLocal from app.pipelines.engine import execute_run async def _inline() -> None: async with SessionLocal() as db: try: # No progress_cb: without Redis there is nobody to publish to. # The engine still writes every step and every progress line to # the run's log, which is what the run screen reads - so an # inline run is just as observable as a queued one. await execute_run(db, run_id) except Exception: log.exception("inline run %s failed", run_id) asyncio.create_task(_inline()) async def request_cancel(run_id: uuid.UUID) -> None: pool = await get_pool() if pool is not None: await pool.set(CANCEL_KEY.format(run_id=run_id), b"1", ex=86400) async def subscribe_events(run_id: uuid.UUID) -> AsyncIterator[dict | None]: """Yield progress events; yield ``None`` as a heartbeat. The heartbeat is what drives the database-polling fallback in the SSE endpoint, so this generator behaves the same with or without Redis. """ pool = await get_pool() if pool is None: while True: yield None await asyncio.sleep(2.0) channel = EVENTS_KEY.format(run_id=run_id) pubsub = pool.pubsub() await pubsub.subscribe(channel) try: while True: # The timeout belongs to `get_message`, which waits on the socket # for it. Wrapping a `get_message()` that defaults to timeout=0 in # `wait_for(..., 2.0)` waits for nothing: it returns None at once, # every time, and the heartbeat this drives - a status re-read # against the database - ran as fast as the event loop could spin # for as long as any run screen was open. try: message = await pubsub.get_message( ignore_subscribe_messages=True, timeout=2.0 ) except TimeoutError: message = None if message and message.get("data"): raw = message["data"] if isinstance(raw, bytes): raw = raw.decode("utf-8", "replace") with contextlib.suppress(json.JSONDecodeError): yield json.loads(raw) continue yield None finally: with contextlib.suppress(Exception): await pubsub.unsubscribe(channel) await pubsub.close()