contiai / apps /api /app /core /queue.py
aliabozahra315's picture
Claude Opus 5
The million Redis commands were a lock over a local file
47885b7
Raw History Blame Contribute Delete
4.47 kB
"""Job queue and live-event plumbing.
Redis is the transport. When it is not reachable the API still works: runs are
executed inline as a background task and progress falls back to polling. That
keeps local development possible without Redis, and keeps a Redis outage from
taking the whole product down.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import uuid
from collections.abc import AsyncIterator
from app.core.config import settings
log = logging.getLogger("contiai.queue")
_pool = None
_pool_failed = False
CANCEL_KEY = "contiai:cancel:{run_id}"
EVENTS_KEY = "contiai:events:{run_id}"
async def get_pool():
"""ARQ pool, or None when Redis is unreachable."""
global _pool, _pool_failed
if _pool is not None or _pool_failed:
return _pool
try:
from arq import create_pool
from arq.connections import RedisSettings
_pool = await create_pool(RedisSettings.from_dsn(settings.redis_url))
return _pool
except Exception as e:
_pool_failed = True
log.warning("Redis unavailable (%s); running jobs inline", e)
return None
def redis_available() -> bool:
"""Whether the queue is live, without attempting a connection.
Read after `get_pool()` has run once at startup. The distinction matters
for reporting: a fallback that works twenty-seven times out of twenty-eight
reads as random flakiness unless the log states which mode the run was in.
"""
return _pool is not None
async def close_pool() -> None:
global _pool
if _pool is not None:
await _pool.close()
_pool = None
async def enqueue_run(run_id: uuid.UUID) -> None:
pool = await get_pool()
if pool is not None:
await pool.enqueue_job("run_pipeline", str(run_id))
return
# No worker available - execute in this process so the run still happens.
from app.db.session import SessionLocal
from app.pipelines.engine import execute_run
async def _inline() -> None:
async with SessionLocal() as db:
try:
# No progress_cb: without Redis there is nobody to publish to.
# The engine still writes every step and every progress line to
# the run's log, which is what the run screen reads - so an
# inline run is just as observable as a queued one.
await execute_run(db, run_id)
except Exception:
log.exception("inline run %s failed", run_id)
asyncio.create_task(_inline())
async def request_cancel(run_id: uuid.UUID) -> None:
pool = await get_pool()
if pool is not None:
await pool.set(CANCEL_KEY.format(run_id=run_id), b"1", ex=86400)
async def subscribe_events(run_id: uuid.UUID) -> AsyncIterator[dict | None]:
"""Yield progress events; yield ``None`` as a heartbeat.
The heartbeat is what drives the database-polling fallback in the SSE
endpoint, so this generator behaves the same with or without Redis.
"""
pool = await get_pool()
if pool is None:
while True:
yield None
await asyncio.sleep(2.0)
channel = EVENTS_KEY.format(run_id=run_id)
pubsub = pool.pubsub()
await pubsub.subscribe(channel)
try:
while True:
# The timeout belongs to `get_message`, which waits on the socket
# for it. Wrapping a `get_message()` that defaults to timeout=0 in
# `wait_for(..., 2.0)` waits for nothing: it returns None at once,
# every time, and the heartbeat this drives - a status re-read
# against the database - ran as fast as the event loop could spin
# for as long as any run screen was open.
try:
message = await pubsub.get_message(
ignore_subscribe_messages=True, timeout=2.0
)
except TimeoutError:
message = None
if message and message.get("data"):
raw = message["data"]
if isinstance(raw, bytes):
raw = raw.decode("utf-8", "replace")
with contextlib.suppress(json.JSONDecodeError):
yield json.loads(raw)
continue
yield None
finally:
with contextlib.suppress(Exception):
await pubsub.unsubscribe(channel)
await pubsub.close()