Download apps/api/app/workers/worker.py from aliroya/contiai: direct link, hf CLI and curl.
- Browser
- Download file 8.36 kB
-
https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/app/workers/worker.py
- Command line
-
hf download hf://spaces/aliroya/contiai/apps/api/app/workers/worker.py
-
curl -L -o worker.py https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/app/workers/worker.py
8.36 kB
| """ARQ worker process. | |
| arq app.workers.worker.WorkerSettings | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import logging | |
| from arq import cron | |
| from arq.connections import RedisSettings | |
| from app.core.config import settings | |
| # Called before anything in a pipeline can print. The ported code logs with | |
| # emoji, and a console that cannot encode one used to take a paid run with it. | |
| from app.core.console import enable_utf8_output | |
| from app.core.runlog import install_tee | |
| from app.db.session import SessionLocal | |
| from app.observability import llm_meter | |
| from app.pipelines.registry import load_all | |
| from app.workers.tasks import rollup_usage, run_pipeline | |
| enable_utf8_output() | |
| install_tee() | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s %(levelname)-5s [%(name)s] %(message)s", | |
| ) | |
| log = logging.getLogger("contiai.worker") | |
| async def startup(ctx: dict) -> None: | |
| # Same decision the API makes in its own startup, and for the same reason: | |
| # CrewAI takes a lock inside `Crew(...)` and reaches for Redis whenever | |
| # REDIS_URL is set. This is the process that builds crews all day, so it | |
| # is the one the lock was costing the most. | |
| from app.core import crew_locks | |
| crew_locks.install() | |
| # Metering has to be installed before any agent module is imported, so a | |
| # call made during a lazy import is still counted. | |
| llm_meter.install() | |
| load_all() | |
| await _release_abandoned_runs(SessionLocal) | |
| ctx["flusher"] = asyncio.create_task(llm_meter.flush_loop(SessionLocal)) | |
| ctx["resumer"] = asyncio.create_task(_resume_flagged_runs(ctx)) | |
| log.info("worker ready: %d pipelines loaded", len(__import__( | |
| "app.pipelines.registry", fromlist=["all_pipelines"]).all_pipelines())) | |
| async def _release_abandoned_runs(session_factory=None) -> None: | |
| """Nothing was running a moment ago, whatever the rows say. | |
| A step's work runs on a thread, and a thread does not take cancellation: | |
| when a fetch inside one stalled, the step outlived its own ninety-minute | |
| limit, ignored a cancel, and held the run `running` for a hundred and sixty | |
| minutes. Redeploying killed the process, and the rows stayed exactly as | |
| they were - the run showed as working, the queue showed it as occupied, and | |
| nothing would ever say otherwise. | |
| This worker has just started, so any row still marked running belongs to a | |
| process that no longer exists. Marked failed, and therefore resumable: a | |
| resumed run replays only the steps that did not succeed. | |
| Marked finished, too. Leaving `finished_at` NULL on a row that is over | |
| reads downstream as «still going»: the screens count from `started_at` to | |
| now when there is no end, so one of these rows showed a duration that grew | |
| for as long as it sat in the list. Now is the honest answer - it is when | |
| we found out, and it is the last moment the row could have been alive. | |
| """ | |
| from datetime import UTC, datetime | |
| from sqlalchemy import update | |
| from app.db.models import Run, RunStatus, StepRun, StepStatus | |
| now = datetime.now(UTC) | |
| # The factory is a parameter for the same reason `flush_once` takes one: | |
| # the test suite runs against its own database, and a reaper that reaches | |
| # for the module-level one would sweep nothing it was asked about. | |
| async with (session_factory or SessionLocal)() as session: | |
| steps = await session.execute( | |
| update(StepRun) | |
| .where(StepRun.status == StepStatus.running) | |
| .values(status=StepStatus.failed, finished_at=now, | |
| error="توقّفت العملية قبل أن تكتمل هذه الخطوة") | |
| ) | |
| runs = await session.execute( | |
| update(Run) | |
| .where(Run.status == RunStatus.running) | |
| .values(status=RunStatus.failed, finished_at=now, | |
| error="توقّفت العملية أثناء التشغيل — يمكن استئنافها") | |
| ) | |
| await session.commit() | |
| if runs.rowcount or steps.rowcount: | |
| log.warning("released %d abandoned run(s) and %d step(s) from a " | |
| "process that no longer exists", runs.rowcount, steps.rowcount) | |
| RESUME_AFTER_BOOT_SECONDS = 60 | |
| async def _resume_flagged_runs(ctx: dict, session_factory=None, delay: float | None = None) -> int: | |
| """A stopped run marked `resume_on_boot` in its params goes on by itself, | |
| once: the deploy that changes its models need not wait for anyone to press | |
| «استئناف». What it finished stays finished; a step's checkpoint is kept. | |
| After a delay, because the API process marks every `running` row failed | |
| when it boots too, and it must not do so to a run already picked up here. | |
| """ | |
| from sqlalchemy import select | |
| from app.db.models import Run, RunStatus, StepRun, StepStatus | |
| await asyncio.sleep(RESUME_AFTER_BOOT_SECONDS if delay is None else delay) | |
| resumed = [] | |
| async with (session_factory or SessionLocal)() as session: | |
| runs = (await session.scalars(select(Run).where(Run.status == RunStatus.failed))).all() | |
| for run in runs: | |
| if not (run.params or {}).get("resume_on_boot"): | |
| continue | |
| run.params = {k: v for k, v in run.params.items() if k != "resume_on_boot"} | |
| steps = (await session.scalars(select(StepRun).where( | |
| StepRun.run_id == run.id, | |
| StepRun.status.in_([StepStatus.failed, StepStatus.running])))).all() | |
| for step in steps: | |
| step.status = StepStatus.pending | |
| step.error = None | |
| run.status = RunStatus.queued | |
| run.error = None | |
| run.finished_at = None | |
| resumed.append(run.id) | |
| await session.commit() | |
| for run_id in resumed: | |
| await ctx["redis"].enqueue_job("run_pipeline", str(run_id)) | |
| log.warning("resumed run %s as its params asked", run_id) | |
| return len(resumed) | |
| async def shutdown(ctx: dict) -> None: | |
| task = ctx.get("flusher") | |
| if task: | |
| task.cancel() | |
| # Do not lose the usage rows still sitting in the queue. | |
| written = await llm_meter.flush_once(SessionLocal) | |
| if written: | |
| log.info("flushed %d usage rows on shutdown", written) | |
| class WorkerSettings: | |
| functions = [run_pipeline] | |
| cron_jobs = [cron(rollup_usage, minute=7)] # hourly, off the hour | |
| on_startup = startup | |
| on_shutdown = shutdown | |
| redis_settings = RedisSettings.from_dsn(settings.redis_url) | |
| # A book can legitimately take this long; the engine enforces per-step | |
| # limits inside it. | |
| job_timeout = settings.run_timeout_seconds | |
| max_jobs = 4 | |
| keep_result = 3600 | |
| # How often an idle worker asks Redis whether there is anything to do. | |
| # | |
| # ARQ's default is half a second, and that one question is what the hosted | |
| # Redis meter actually measures: `_poll_iteration` issues a single | |
| # `zrangebyscore` per pass, so the default costs two commands a second - | |
| # 7,200 an hour, 172,800 for every day the container stays awake. On a | |
| # free tier of 500,000 a month the dashboard read 329,000 after two days | |
| # and two books, and when that runs out Redis refuses commands and the | |
| # queue stops accepting work: the limit arrives as a stop, not as a | |
| # slowdown. | |
| # | |
| # Fifteen seconds is 240 an hour: 172,800 for a month spent entirely | |
| # awake, a third of the allowance, leaving the rest for the traffic that | |
| # is actually work - enqueueing, results, the progress channel, the cancel | |
| # keys. Five was tried first and does not fit at all (518,400 against | |
| # 500,000) and ten leaves no margin (259,200). The worst case is the one | |
| # to design for when the failure mode is the queue refusing jobs. | |
| # | |
| # What it costs is that a queued run may sit up to fifteen seconds before | |
| # the worker picks it up - against a book that takes twenty to forty | |
| # minutes to write, and a progress screen that polls the database, not | |
| # Redis. | |
| # | |
| # `allow_abort_jobs` is left alone: ARQ defaults it off, and cancelling a | |
| # run here goes through `queue.request_cancel`, which sets its own key that | |
| # the engine reads between steps. Turning it on would add two commands to | |
| # every one of these polls for a mechanism this system does not use. | |
| poll_delay = 15 | |