File size: 4,473 Bytes
57fa15f 47885b7 57fa15f 47885b7 57fa15f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | """Job queue and live-event plumbing.
Redis is the transport. When it is not reachable the API still works: runs are
executed inline as a background task and progress falls back to polling. That
keeps local development possible without Redis, and keeps a Redis outage from
taking the whole product down.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import uuid
from collections.abc import AsyncIterator
from app.core.config import settings
log = logging.getLogger("contiai.queue")
_pool = None
_pool_failed = False
CANCEL_KEY = "contiai:cancel:{run_id}"
EVENTS_KEY = "contiai:events:{run_id}"
async def get_pool():
"""ARQ pool, or None when Redis is unreachable."""
global _pool, _pool_failed
if _pool is not None or _pool_failed:
return _pool
try:
from arq import create_pool
from arq.connections import RedisSettings
_pool = await create_pool(RedisSettings.from_dsn(settings.redis_url))
return _pool
except Exception as e:
_pool_failed = True
log.warning("Redis unavailable (%s); running jobs inline", e)
return None
def redis_available() -> bool:
"""Whether the queue is live, without attempting a connection.
Read after `get_pool()` has run once at startup. The distinction matters
for reporting: a fallback that works twenty-seven times out of twenty-eight
reads as random flakiness unless the log states which mode the run was in.
"""
return _pool is not None
async def close_pool() -> None:
global _pool
if _pool is not None:
await _pool.close()
_pool = None
async def enqueue_run(run_id: uuid.UUID) -> None:
pool = await get_pool()
if pool is not None:
await pool.enqueue_job("run_pipeline", str(run_id))
return
# No worker available - execute in this process so the run still happens.
from app.db.session import SessionLocal
from app.pipelines.engine import execute_run
async def _inline() -> None:
async with SessionLocal() as db:
try:
# No progress_cb: without Redis there is nobody to publish to.
# The engine still writes every step and every progress line to
# the run's log, which is what the run screen reads - so an
# inline run is just as observable as a queued one.
await execute_run(db, run_id)
except Exception:
log.exception("inline run %s failed", run_id)
asyncio.create_task(_inline())
async def request_cancel(run_id: uuid.UUID) -> None:
pool = await get_pool()
if pool is not None:
await pool.set(CANCEL_KEY.format(run_id=run_id), b"1", ex=86400)
async def subscribe_events(run_id: uuid.UUID) -> AsyncIterator[dict | None]:
"""Yield progress events; yield ``None`` as a heartbeat.
The heartbeat is what drives the database-polling fallback in the SSE
endpoint, so this generator behaves the same with or without Redis.
"""
pool = await get_pool()
if pool is None:
while True:
yield None
await asyncio.sleep(2.0)
channel = EVENTS_KEY.format(run_id=run_id)
pubsub = pool.pubsub()
await pubsub.subscribe(channel)
try:
while True:
# The timeout belongs to `get_message`, which waits on the socket
# for it. Wrapping a `get_message()` that defaults to timeout=0 in
# `wait_for(..., 2.0)` waits for nothing: it returns None at once,
# every time, and the heartbeat this drives - a status re-read
# against the database - ran as fast as the event loop could spin
# for as long as any run screen was open.
try:
message = await pubsub.get_message(
ignore_subscribe_messages=True, timeout=2.0
)
except TimeoutError:
message = None
if message and message.get("data"):
raw = message["data"]
if isinstance(raw, bytes):
raw = raw.decode("utf-8", "replace")
with contextlib.suppress(json.JSONDecodeError):
yield json.loads(raw)
continue
yield None
finally:
with contextlib.suppress(Exception):
await pubsub.unsubscribe(channel)
await pubsub.close()
|