Download apps/api/scripts/ab_writer.py from aliroya/contiai: direct link, hf CLI and curl.
- Browser
- Download file 5.54 kB
-
https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/scripts/ab_writer.py
- Command line
-
hf download hf://spaces/aliroya/contiai/apps/api/scripts/ab_writer.py
-
curl -L -o ab_writer.py https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/scripts/ab_writer.py
5.54 kB
| """Two writer models on the same real محاور: pass rate, calls and cost. | |
| The writer is the bill - 92% of a hundred-محور bag - and the model the | |
| notebook tuned against is not the cheapest that writes this contract. But | |
| cost is price times calls: a cheaper model the validator sends back twice as | |
| often saves nothing. So before the setting changes, the two are run on the | |
| same plans with the same sources, and the figures are read side by side. | |
| python scripts/ab_writer.py --run <run id> --covers 10 \ | |
| --models anthropic/claude-haiku-4.5,google/gemini-2.5-flash | |
| Spends tokens - about half a dollar for ten محاور on two models. Reads the | |
| run's stored outputs; writes nothing to the database (the meter's flush loop | |
| is not started, so no `llm_calls` rows are added either). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import sys | |
| import time | |
| import uuid | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from sqlalchemy import select # noqa: E402 | |
| from app.agents.arabic_bag.context import using # noqa: E402 | |
| from app.core.config import settings # noqa: E402 | |
| from app.db.models import Run, StepRun, StepStatus # noqa: E402 | |
| from app.db.session import SessionLocal # noqa: E402 | |
| from app.observability import llm_meter # noqa: E402 | |
| from app.observability.llm_meter import llm_context # noqa: E402 | |
| DEFAULT_MODELS = "anthropic/claude-haiku-4.5,google/gemini-2.5-flash" | |
| async def _load(run_id: str | None) -> tuple[Run, dict[str, dict]]: | |
| async with SessionLocal() as db: | |
| if run_id: | |
| run = await db.get(Run, uuid.UUID(run_id)) | |
| else: | |
| run = await db.scalar( | |
| select(Run).where(Run.pipeline_key == "arabic_training_bag", | |
| Run.status == "succeeded") | |
| .order_by(Run.queued_at.desc()).limit(1) | |
| ) | |
| if run is None: | |
| raise SystemExit("no such run") | |
| rows = (await db.scalars( | |
| select(StepRun).where(StepRun.run_id == run.id, | |
| StepRun.status == StepStatus.succeeded) | |
| )).all() | |
| return run, {r.step_key: (r.output or {}) for r in rows} | |
| def _spread(items: list, n: int) -> list: | |
| """`n` items spread over the list - every day and session represented.""" | |
| if n >= len(items): | |
| return list(items) | |
| step = len(items) / n | |
| return [items[int(i * step)] for i in range(n)] | |
| async def _write(model: str, blueprint: dict, plans: list, scraped: list, | |
| credits: dict) -> dict: | |
| from app.agents.arabic_bag import llms | |
| from app.agents.arabic_bag.writer import write_one_cover | |
| settings.bag_writer_model = model | |
| llms._available = None # re-read the catalogue for the new name | |
| tag = f"ab-{model.replace('/', '-')}" | |
| llm_meter.seed_spent(tag, 0.0) | |
| passed = calls = words = 0 | |
| started = time.time() | |
| with llm_context(run_id=tag, step_run_id=None, user_id=None, | |
| pipeline_key="ab_writer", step_key="bag_ar.content"): | |
| with using(blueprint): | |
| prev: list[str] = [] | |
| for i, plan in enumerate(plans, 1): | |
| print(f"\n[{model}] {i}/{len(plans)} {str(plan['cover'])[:48]}") | |
| res = await write_one_cover(plan, scraped, prev, credits=credits) | |
| calls += int(res.get("_calls") or 0) | |
| passed += 0 if res.get("_validation_failed") else 1 | |
| words += sum(len(str(p).split()) for b in res.get("blocks", []) | |
| for p in (b.get("paragraphs") or [])) | |
| prev.append(f"{plan['cover']}: تم تناوله") | |
| return {"model": model, "passed": passed, "calls": calls, | |
| "usd": llm_meter.spent(tag), "seconds": time.time() - started, | |
| "words": words} | |
| async def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) | |
| parser.add_argument("--run", help="run id; the latest finished bag when omitted") | |
| parser.add_argument("--covers", type=int, default=10) | |
| parser.add_argument("--models", default=DEFAULT_MODELS) | |
| args = parser.parse_args() | |
| llm_meter.install() | |
| llm_meter.ensure() | |
| run, outputs = await _load(args.run) | |
| blueprint = outputs.get("bag_ar.blueprint") or {} | |
| plans = (outputs.get("bag_ar.plan") or {}).get("plans") or [] | |
| scraped = (outputs.get("bag_ar.sources") or {}).get("value") or [] | |
| credits = (outputs.get("bag_ar.citations") or {}).get("value") or {} | |
| if not plans: | |
| raise SystemExit("the run has no plans to write from") | |
| chosen = _spread(plans, args.covers) | |
| print(f"run {run.id}: {len(chosen)} of {len(plans)} محاور, " | |
| f"{len(args.models.split(','))} models") | |
| results = [] | |
| for model in [m.strip() for m in args.models.split(",") if m.strip()]: | |
| results.append(await _write(model, blueprint, chosen, scraped, credits)) | |
| print("\n" + "=" * 78) | |
| print(f"{'model':36} {'pass':>5} {'calls':>6} {'$':>7} {'$/محور':>8} {'min':>5}") | |
| for r in results: | |
| n = len(chosen) | |
| print(f"{r['model']:36} {r['passed']:>3}/{n:<2} {r['calls']:>6} " | |
| f"{r['usd']:>7.3f} {r['usd'] / n:>8.4f} {r['seconds'] / 60:>5.1f}") | |
| print("=" * 78) | |
| print("القرار: الأرخص للمحور الواحد بنسبة نجاح مقاربة؛ الأرخص الذي يُرفض " | |
| "ضعف الآخر لا يوفّر شيئًا.") | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |