File size: 8,359 Bytes
b688fe8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47885b7
 
b688fe8
 
47885b7
b688fe8
 
 
 
 
68bf31d
b688fe8
630ed56
b688fe8
 
 
 
68bf31d
 
 
 
 
 
 
 
 
 
 
 
 
4ad0ad3
 
 
 
 
 
68bf31d
4ad0ad3
 
68bf31d
 
 
 
4ad0ad3
 
68bf31d
 
 
 
 
 
 
4ad0ad3
68bf31d
 
 
 
 
4ad0ad3
68bf31d
 
 
 
 
 
 
 
 
630ed56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b688fe8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7d01796
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
"""ARQ worker process.

    arq app.workers.worker.WorkerSettings
"""

from __future__ import annotations

import asyncio
import logging

from arq import cron
from arq.connections import RedisSettings

from app.core.config import settings

# Called before anything in a pipeline can print. The ported code logs with
# emoji, and a console that cannot encode one used to take a paid run with it.
from app.core.console import enable_utf8_output
from app.core.runlog import install_tee
from app.db.session import SessionLocal
from app.observability import llm_meter
from app.pipelines.registry import load_all
from app.workers.tasks import rollup_usage, run_pipeline

enable_utf8_output()
install_tee()

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s %(levelname)-5s [%(name)s] %(message)s",
)
log = logging.getLogger("contiai.worker")


async def startup(ctx: dict) -> None:
    # Same decision the API makes in its own startup, and for the same reason:
    # CrewAI takes a lock inside `Crew(...)` and reaches for Redis whenever
    # REDIS_URL is set. This is the process that builds crews all day, so it
    # is the one the lock was costing the most.
    from app.core import crew_locks

    crew_locks.install()

    # Metering has to be installed before any agent module is imported, so a
    # call made during a lazy import is still counted.
    llm_meter.install()
    load_all()
    await _release_abandoned_runs(SessionLocal)
    ctx["flusher"] = asyncio.create_task(llm_meter.flush_loop(SessionLocal))
    ctx["resumer"] = asyncio.create_task(_resume_flagged_runs(ctx))
    log.info("worker ready: %d pipelines loaded", len(__import__(
        "app.pipelines.registry", fromlist=["all_pipelines"]).all_pipelines()))


async def _release_abandoned_runs(session_factory=None) -> None:
    """Nothing was running a moment ago, whatever the rows say.

    A step's work runs on a thread, and a thread does not take cancellation:
    when a fetch inside one stalled, the step outlived its own ninety-minute
    limit, ignored a cancel, and held the run `running` for a hundred and sixty
    minutes. Redeploying killed the process, and the rows stayed exactly as
    they were - the run showed as working, the queue showed it as occupied, and
    nothing would ever say otherwise.

    This worker has just started, so any row still marked running belongs to a
    process that no longer exists. Marked failed, and therefore resumable: a
    resumed run replays only the steps that did not succeed.

    Marked finished, too. Leaving `finished_at` NULL on a row that is over
    reads downstream as «still going»: the screens count from `started_at` to
    now when there is no end, so one of these rows showed a duration that grew
    for as long as it sat in the list. Now is the honest answer - it is when
    we found out, and it is the last moment the row could have been alive.
    """
    from datetime import UTC, datetime

    from sqlalchemy import update

    from app.db.models import Run, RunStatus, StepRun, StepStatus

    now = datetime.now(UTC)

    # The factory is a parameter for the same reason `flush_once` takes one:
    # the test suite runs against its own database, and a reaper that reaches
    # for the module-level one would sweep nothing it was asked about.
    async with (session_factory or SessionLocal)() as session:
        steps = await session.execute(
            update(StepRun)
            .where(StepRun.status == StepStatus.running)
            .values(status=StepStatus.failed, finished_at=now,
                    error="توقّفت العملية قبل أن تكتمل هذه الخطوة")
        )
        runs = await session.execute(
            update(Run)
            .where(Run.status == RunStatus.running)
            .values(status=RunStatus.failed, finished_at=now,
                    error="توقّفت العملية أثناء التشغيل — يمكن استئنافها")
        )
        await session.commit()

    if runs.rowcount or steps.rowcount:
        log.warning("released %d abandoned run(s) and %d step(s) from a "
                    "process that no longer exists", runs.rowcount, steps.rowcount)


RESUME_AFTER_BOOT_SECONDS = 60


async def _resume_flagged_runs(ctx: dict, session_factory=None, delay: float | None = None) -> int:
    """A stopped run marked `resume_on_boot` in its params goes on by itself,
    once: the deploy that changes its models need not wait for anyone to press
    «استئناف». What it finished stays finished; a step's checkpoint is kept.

    After a delay, because the API process marks every `running` row failed
    when it boots too, and it must not do so to a run already picked up here.
    """
    from sqlalchemy import select

    from app.db.models import Run, RunStatus, StepRun, StepStatus

    await asyncio.sleep(RESUME_AFTER_BOOT_SECONDS if delay is None else delay)
    resumed = []
    async with (session_factory or SessionLocal)() as session:
        runs = (await session.scalars(select(Run).where(Run.status == RunStatus.failed))).all()
        for run in runs:
            if not (run.params or {}).get("resume_on_boot"):
                continue
            run.params = {k: v for k, v in run.params.items() if k != "resume_on_boot"}
            steps = (await session.scalars(select(StepRun).where(
                StepRun.run_id == run.id,
                StepRun.status.in_([StepStatus.failed, StepStatus.running])))).all()
            for step in steps:
                step.status = StepStatus.pending
                step.error = None
            run.status = RunStatus.queued
            run.error = None
            run.finished_at = None
            resumed.append(run.id)
        await session.commit()
    for run_id in resumed:
        await ctx["redis"].enqueue_job("run_pipeline", str(run_id))
        log.warning("resumed run %s as its params asked", run_id)
    return len(resumed)


async def shutdown(ctx: dict) -> None:
    task = ctx.get("flusher")
    if task:
        task.cancel()
    # Do not lose the usage rows still sitting in the queue.
    written = await llm_meter.flush_once(SessionLocal)
    if written:
        log.info("flushed %d usage rows on shutdown", written)


class WorkerSettings:
    functions = [run_pipeline]
    cron_jobs = [cron(rollup_usage, minute=7)]  # hourly, off the hour
    on_startup = startup
    on_shutdown = shutdown
    redis_settings = RedisSettings.from_dsn(settings.redis_url)
    # A book can legitimately take this long; the engine enforces per-step
    # limits inside it.
    job_timeout = settings.run_timeout_seconds
    max_jobs = 4
    keep_result = 3600

    # How often an idle worker asks Redis whether there is anything to do.
    #
    # ARQ's default is half a second, and that one question is what the hosted
    # Redis meter actually measures: `_poll_iteration` issues a single
    # `zrangebyscore` per pass, so the default costs two commands a second -
    # 7,200 an hour, 172,800 for every day the container stays awake. On a
    # free tier of 500,000 a month the dashboard read 329,000 after two days
    # and two books, and when that runs out Redis refuses commands and the
    # queue stops accepting work: the limit arrives as a stop, not as a
    # slowdown.
    #
    # Fifteen seconds is 240 an hour: 172,800 for a month spent entirely
    # awake, a third of the allowance, leaving the rest for the traffic that
    # is actually work - enqueueing, results, the progress channel, the cancel
    # keys. Five was tried first and does not fit at all (518,400 against
    # 500,000) and ten leaves no margin (259,200). The worst case is the one
    # to design for when the failure mode is the queue refusing jobs.
    #
    # What it costs is that a queued run may sit up to fifteen seconds before
    # the worker picks it up - against a book that takes twenty to forty
    # minutes to write, and a progress screen that polls the database, not
    # Redis.
    #
    # `allow_abort_jobs` is left alone: ARQ defaults it off, and cancelling a
    # run here goes through `queue.request_cancel`, which sets its own key that
    # the engine reads between steps. Turning it on would add two commands to
    # every one of these polls for a mechanism this system does not use.
    poll_delay = 15