File size: 4,176 Bytes
2bb5af3 4ad0ad3 2bb5af3 4ad0ad3 2bb5af3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | """Background jobs.
Generating a book takes minutes, sometimes tens of minutes. The old system did
it inside the HTTP request, which meant no progress, no recovery and a hard
ceiling from the proxy's timeout. Here the API enqueues and returns; the worker
does the work and writes progress to the database as it goes.
"""
from __future__ import annotations
import logging
import uuid
from datetime import UTC, datetime
from sqlalchemy import select
from app.db.models import Run, RunStatus
from app.db.session import SessionLocal
from app.pipelines.engine import execute_run
from app.pipelines.registry import load_all
log = logging.getLogger("contiai.worker")
# Key in Redis that the API sets to ask a running job to stop.
CANCEL_KEY = "contiai:cancel:{run_id}"
# Channel the SSE endpoint subscribes to for live progress.
EVENTS_KEY = "contiai:events:{run_id}"
async def run_pipeline(ctx: dict, run_id: str) -> dict:
"""ARQ entrypoint."""
load_all()
rid = uuid.UUID(run_id)
redis = ctx.get("redis")
async def progress(step_key: str, message: str, percent: float | None) -> None:
if redis is None:
return
import json
await redis.publish(
EVENTS_KEY.format(run_id=run_id),
json.dumps({"step": step_key, "message": message, "percent": percent}),
)
async def cancelled() -> bool:
if redis is None:
return False
return bool(await redis.get(CANCEL_KEY.format(run_id=run_id)))
async with SessionLocal() as db:
try:
status = await execute_run(db, rid, progress_cb=progress, cancel_cb=cancelled)
except Exception as e:
log.exception("run %s crashed outside the engine", run_id)
run = await db.get(Run, rid)
if run is not None:
run.status = RunStatus.failed
run.error = f"{type(e).__name__}: {e}"
# The engine writes this on every ending it controls; this is
# the ending it does not. Without it the run reads as still
# running and its duration keeps climbing on the screen.
run.finished_at = datetime.now(UTC)
await db.commit()
raise
if redis is not None:
await redis.delete(CANCEL_KEY.format(run_id=run_id))
return {"run_id": run_id, "status": status.value}
async def rollup_usage(ctx: dict) -> dict:
"""Fold ``llm_calls`` into ``usage_daily``.
The dashboard charts read the rollup, so a year of history stays a cheap
query no matter how many individual calls were made.
"""
from sqlalchemy import func, text
async with SessionLocal() as db:
# Written as one statement so the rollup is atomic and re-runnable.
await db.execute(
text(
"""
insert into usage_daily (
id, user_id, day, total_tokens, prompt_tokens,
completion_tokens, cost_usd, runs_count, calls_count
)
select
gen_random_uuid(),
user_id,
date(created_at) as day,
sum(total_tokens),
sum(prompt_tokens),
sum(completion_tokens),
sum(cost_usd),
count(distinct run_id),
count(*)
from llm_calls
where user_id is not null
group by user_id, date(created_at)
on conflict (user_id, day) do update set
total_tokens = excluded.total_tokens,
prompt_tokens = excluded.prompt_tokens,
completion_tokens = excluded.completion_tokens,
cost_usd = excluded.cost_usd,
runs_count = excluded.runs_count,
calls_count = excluded.calls_count
"""
)
)
await db.commit()
total = await db.scalar(select(func.count()).select_from(text("usage_daily")))
return {"usage_daily_rows": int(total or 0)}
|