contiai / apps /api /app /workers /worker.py
aliabozahra315's picture
Deploy 2c8cfb1
630ed56
Raw History Blame Contribute Delete
8.36 kB
"""ARQ worker process.
arq app.workers.worker.WorkerSettings
"""
from __future__ import annotations
import asyncio
import logging
from arq import cron
from arq.connections import RedisSettings
from app.core.config import settings
# Called before anything in a pipeline can print. The ported code logs with
# emoji, and a console that cannot encode one used to take a paid run with it.
from app.core.console import enable_utf8_output
from app.core.runlog import install_tee
from app.db.session import SessionLocal
from app.observability import llm_meter
from app.pipelines.registry import load_all
from app.workers.tasks import rollup_usage, run_pipeline
enable_utf8_output()
install_tee()
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)-5s [%(name)s] %(message)s",
)
log = logging.getLogger("contiai.worker")
async def startup(ctx: dict) -> None:
# Same decision the API makes in its own startup, and for the same reason:
# CrewAI takes a lock inside `Crew(...)` and reaches for Redis whenever
# REDIS_URL is set. This is the process that builds crews all day, so it
# is the one the lock was costing the most.
from app.core import crew_locks
crew_locks.install()
# Metering has to be installed before any agent module is imported, so a
# call made during a lazy import is still counted.
llm_meter.install()
load_all()
await _release_abandoned_runs(SessionLocal)
ctx["flusher"] = asyncio.create_task(llm_meter.flush_loop(SessionLocal))
ctx["resumer"] = asyncio.create_task(_resume_flagged_runs(ctx))
log.info("worker ready: %d pipelines loaded", len(__import__(
"app.pipelines.registry", fromlist=["all_pipelines"]).all_pipelines()))
async def _release_abandoned_runs(session_factory=None) -> None:
"""Nothing was running a moment ago, whatever the rows say.
A step's work runs on a thread, and a thread does not take cancellation:
when a fetch inside one stalled, the step outlived its own ninety-minute
limit, ignored a cancel, and held the run `running` for a hundred and sixty
minutes. Redeploying killed the process, and the rows stayed exactly as
they were - the run showed as working, the queue showed it as occupied, and
nothing would ever say otherwise.
This worker has just started, so any row still marked running belongs to a
process that no longer exists. Marked failed, and therefore resumable: a
resumed run replays only the steps that did not succeed.
Marked finished, too. Leaving `finished_at` NULL on a row that is over
reads downstream as «still going»: the screens count from `started_at` to
now when there is no end, so one of these rows showed a duration that grew
for as long as it sat in the list. Now is the honest answer - it is when
we found out, and it is the last moment the row could have been alive.
"""
from datetime import UTC, datetime
from sqlalchemy import update
from app.db.models import Run, RunStatus, StepRun, StepStatus
now = datetime.now(UTC)
# The factory is a parameter for the same reason `flush_once` takes one:
# the test suite runs against its own database, and a reaper that reaches
# for the module-level one would sweep nothing it was asked about.
async with (session_factory or SessionLocal)() as session:
steps = await session.execute(
update(StepRun)
.where(StepRun.status == StepStatus.running)
.values(status=StepStatus.failed, finished_at=now,
error="توقّفت العملية قبل أن تكتمل هذه الخطوة")
)
runs = await session.execute(
update(Run)
.where(Run.status == RunStatus.running)
.values(status=RunStatus.failed, finished_at=now,
error="توقّفت العملية أثناء التشغيل — يمكن استئنافها")
)
await session.commit()
if runs.rowcount or steps.rowcount:
log.warning("released %d abandoned run(s) and %d step(s) from a "
"process that no longer exists", runs.rowcount, steps.rowcount)
RESUME_AFTER_BOOT_SECONDS = 60
async def _resume_flagged_runs(ctx: dict, session_factory=None, delay: float | None = None) -> int:
"""A stopped run marked `resume_on_boot` in its params goes on by itself,
once: the deploy that changes its models need not wait for anyone to press
«استئناف». What it finished stays finished; a step's checkpoint is kept.
After a delay, because the API process marks every `running` row failed
when it boots too, and it must not do so to a run already picked up here.
"""
from sqlalchemy import select
from app.db.models import Run, RunStatus, StepRun, StepStatus
await asyncio.sleep(RESUME_AFTER_BOOT_SECONDS if delay is None else delay)
resumed = []
async with (session_factory or SessionLocal)() as session:
runs = (await session.scalars(select(Run).where(Run.status == RunStatus.failed))).all()
for run in runs:
if not (run.params or {}).get("resume_on_boot"):
continue
run.params = {k: v for k, v in run.params.items() if k != "resume_on_boot"}
steps = (await session.scalars(select(StepRun).where(
StepRun.run_id == run.id,
StepRun.status.in_([StepStatus.failed, StepStatus.running])))).all()
for step in steps:
step.status = StepStatus.pending
step.error = None
run.status = RunStatus.queued
run.error = None
run.finished_at = None
resumed.append(run.id)
await session.commit()
for run_id in resumed:
await ctx["redis"].enqueue_job("run_pipeline", str(run_id))
log.warning("resumed run %s as its params asked", run_id)
return len(resumed)
async def shutdown(ctx: dict) -> None:
task = ctx.get("flusher")
if task:
task.cancel()
# Do not lose the usage rows still sitting in the queue.
written = await llm_meter.flush_once(SessionLocal)
if written:
log.info("flushed %d usage rows on shutdown", written)
class WorkerSettings:
functions = [run_pipeline]
cron_jobs = [cron(rollup_usage, minute=7)] # hourly, off the hour
on_startup = startup
on_shutdown = shutdown
redis_settings = RedisSettings.from_dsn(settings.redis_url)
# A book can legitimately take this long; the engine enforces per-step
# limits inside it.
job_timeout = settings.run_timeout_seconds
max_jobs = 4
keep_result = 3600
# How often an idle worker asks Redis whether there is anything to do.
#
# ARQ's default is half a second, and that one question is what the hosted
# Redis meter actually measures: `_poll_iteration` issues a single
# `zrangebyscore` per pass, so the default costs two commands a second -
# 7,200 an hour, 172,800 for every day the container stays awake. On a
# free tier of 500,000 a month the dashboard read 329,000 after two days
# and two books, and when that runs out Redis refuses commands and the
# queue stops accepting work: the limit arrives as a stop, not as a
# slowdown.
#
# Fifteen seconds is 240 an hour: 172,800 for a month spent entirely
# awake, a third of the allowance, leaving the rest for the traffic that
# is actually work - enqueueing, results, the progress channel, the cancel
# keys. Five was tried first and does not fit at all (518,400 against
# 500,000) and ten leaves no margin (259,200). The worst case is the one
# to design for when the failure mode is the queue refusing jobs.
#
# What it costs is that a queued run may sit up to fifteen seconds before
# the worker picks it up - against a book that takes twenty to forty
# minutes to write, and a progress screen that polls the database, not
# Redis.
#
# `allow_abort_jobs` is left alone: ARQ defaults it off, and cancelling a
# run here goes through `queue.request_cancel`, which sets its own key that
# the engine reads between steps. Turning it on would add two commands to
# every one of these polls for a mechanism this system does not use.
poll_delay = 15