"""Two writer models on the same real محاور: pass rate, calls and cost. The writer is the bill - 92% of a hundred-محور bag - and the model the notebook tuned against is not the cheapest that writes this contract. But cost is price times calls: a cheaper model the validator sends back twice as often saves nothing. So before the setting changes, the two are run on the same plans with the same sources, and the figures are read side by side. python scripts/ab_writer.py --run --covers 10 \ --models anthropic/claude-haiku-4.5,google/gemini-2.5-flash Spends tokens - about half a dollar for ten محاور on two models. Reads the run's stored outputs; writes nothing to the database (the meter's flush loop is not started, so no `llm_calls` rows are added either). """ from __future__ import annotations import argparse import asyncio import sys import time import uuid from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from sqlalchemy import select # noqa: E402 from app.agents.arabic_bag.context import using # noqa: E402 from app.core.config import settings # noqa: E402 from app.db.models import Run, StepRun, StepStatus # noqa: E402 from app.db.session import SessionLocal # noqa: E402 from app.observability import llm_meter # noqa: E402 from app.observability.llm_meter import llm_context # noqa: E402 DEFAULT_MODELS = "anthropic/claude-haiku-4.5,google/gemini-2.5-flash" async def _load(run_id: str | None) -> tuple[Run, dict[str, dict]]: async with SessionLocal() as db: if run_id: run = await db.get(Run, uuid.UUID(run_id)) else: run = await db.scalar( select(Run).where(Run.pipeline_key == "arabic_training_bag", Run.status == "succeeded") .order_by(Run.queued_at.desc()).limit(1) ) if run is None: raise SystemExit("no such run") rows = (await db.scalars( select(StepRun).where(StepRun.run_id == run.id, StepRun.status == StepStatus.succeeded) )).all() return run, {r.step_key: (r.output or {}) for r in rows} def _spread(items: list, n: int) -> list: """`n` items spread over the list - every day and session represented.""" if n >= len(items): return list(items) step = len(items) / n return [items[int(i * step)] for i in range(n)] async def _write(model: str, blueprint: dict, plans: list, scraped: list, credits: dict) -> dict: from app.agents.arabic_bag import llms from app.agents.arabic_bag.writer import write_one_cover settings.bag_writer_model = model llms._available = None # re-read the catalogue for the new name tag = f"ab-{model.replace('/', '-')}" llm_meter.seed_spent(tag, 0.0) passed = calls = words = 0 started = time.time() with llm_context(run_id=tag, step_run_id=None, user_id=None, pipeline_key="ab_writer", step_key="bag_ar.content"): with using(blueprint): prev: list[str] = [] for i, plan in enumerate(plans, 1): print(f"\n[{model}] {i}/{len(plans)} {str(plan['cover'])[:48]}") res = await write_one_cover(plan, scraped, prev, credits=credits) calls += int(res.get("_calls") or 0) passed += 0 if res.get("_validation_failed") else 1 words += sum(len(str(p).split()) for b in res.get("blocks", []) for p in (b.get("paragraphs") or [])) prev.append(f"{plan['cover']}: تم تناوله") return {"model": model, "passed": passed, "calls": calls, "usd": llm_meter.spent(tag), "seconds": time.time() - started, "words": words} async def main() -> None: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--run", help="run id; the latest finished bag when omitted") parser.add_argument("--covers", type=int, default=10) parser.add_argument("--models", default=DEFAULT_MODELS) args = parser.parse_args() llm_meter.install() llm_meter.ensure() run, outputs = await _load(args.run) blueprint = outputs.get("bag_ar.blueprint") or {} plans = (outputs.get("bag_ar.plan") or {}).get("plans") or [] scraped = (outputs.get("bag_ar.sources") or {}).get("value") or [] credits = (outputs.get("bag_ar.citations") or {}).get("value") or {} if not plans: raise SystemExit("the run has no plans to write from") chosen = _spread(plans, args.covers) print(f"run {run.id}: {len(chosen)} of {len(plans)} محاور, " f"{len(args.models.split(','))} models") results = [] for model in [m.strip() for m in args.models.split(",") if m.strip()]: results.append(await _write(model, blueprint, chosen, scraped, credits)) print("\n" + "=" * 78) print(f"{'model':36} {'pass':>5} {'calls':>6} {'$':>7} {'$/محور':>8} {'min':>5}") for r in results: n = len(chosen) print(f"{r['model']:36} {r['passed']:>3}/{n:<2} {r['calls']:>6} " f"{r['usd']:>7.3f} {r['usd'] / n:>8.4f} {r['seconds'] / 60:>5.1f}") print("=" * 78) print("القرار: الأرخص للمحور الواحد بنسبة نجاح مقاربة؛ الأرخص الذي يُرفض " "ضعف الآخر لا يوفّر شيئًا.") if __name__ == "__main__": asyncio.run(main())