File size: 4,473 Bytes
57fa15f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47885b7
 
 
 
 
 
57fa15f
47885b7
 
57fa15f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
"""Job queue and live-event plumbing.

Redis is the transport. When it is not reachable the API still works: runs are
executed inline as a background task and progress falls back to polling. That
keeps local development possible without Redis, and keeps a Redis outage from
taking the whole product down.
"""

from __future__ import annotations

import asyncio
import contextlib
import json
import logging
import uuid
from collections.abc import AsyncIterator

from app.core.config import settings

log = logging.getLogger("contiai.queue")

_pool = None
_pool_failed = False

CANCEL_KEY = "contiai:cancel:{run_id}"
EVENTS_KEY = "contiai:events:{run_id}"


async def get_pool():
    """ARQ pool, or None when Redis is unreachable."""
    global _pool, _pool_failed
    if _pool is not None or _pool_failed:
        return _pool
    try:
        from arq import create_pool
        from arq.connections import RedisSettings

        _pool = await create_pool(RedisSettings.from_dsn(settings.redis_url))
        return _pool
    except Exception as e:
        _pool_failed = True
        log.warning("Redis unavailable (%s); running jobs inline", e)
        return None


def redis_available() -> bool:
    """Whether the queue is live, without attempting a connection.

    Read after `get_pool()` has run once at startup. The distinction matters
    for reporting: a fallback that works twenty-seven times out of twenty-eight
    reads as random flakiness unless the log states which mode the run was in.
    """
    return _pool is not None


async def close_pool() -> None:
    global _pool
    if _pool is not None:
        await _pool.close()
        _pool = None


async def enqueue_run(run_id: uuid.UUID) -> None:
    pool = await get_pool()
    if pool is not None:
        await pool.enqueue_job("run_pipeline", str(run_id))
        return

    # No worker available - execute in this process so the run still happens.
    from app.db.session import SessionLocal
    from app.pipelines.engine import execute_run

    async def _inline() -> None:
        async with SessionLocal() as db:
            try:
                # No progress_cb: without Redis there is nobody to publish to.
                # The engine still writes every step and every progress line to
                # the run's log, which is what the run screen reads - so an
                # inline run is just as observable as a queued one.
                await execute_run(db, run_id)
            except Exception:
                log.exception("inline run %s failed", run_id)

    asyncio.create_task(_inline())


async def request_cancel(run_id: uuid.UUID) -> None:
    pool = await get_pool()
    if pool is not None:
        await pool.set(CANCEL_KEY.format(run_id=run_id), b"1", ex=86400)


async def subscribe_events(run_id: uuid.UUID) -> AsyncIterator[dict | None]:
    """Yield progress events; yield ``None`` as a heartbeat.

    The heartbeat is what drives the database-polling fallback in the SSE
    endpoint, so this generator behaves the same with or without Redis.
    """
    pool = await get_pool()
    if pool is None:
        while True:
            yield None
            await asyncio.sleep(2.0)

    channel = EVENTS_KEY.format(run_id=run_id)
    pubsub = pool.pubsub()
    await pubsub.subscribe(channel)
    try:
        while True:
            # The timeout belongs to `get_message`, which waits on the socket
            # for it. Wrapping a `get_message()` that defaults to timeout=0 in
            # `wait_for(..., 2.0)` waits for nothing: it returns None at once,
            # every time, and the heartbeat this drives - a status re-read
            # against the database - ran as fast as the event loop could spin
            # for as long as any run screen was open.
            try:
                message = await pubsub.get_message(
                    ignore_subscribe_messages=True, timeout=2.0
                )
            except TimeoutError:
                message = None

            if message and message.get("data"):
                raw = message["data"]
                if isinstance(raw, bytes):
                    raw = raw.decode("utf-8", "replace")
                with contextlib.suppress(json.JSONDecodeError):
                    yield json.loads(raw)
                    continue
            yield None
    finally:
        with contextlib.suppress(Exception):
            await pubsub.unsubscribe(channel)
            await pubsub.close()