File size: 5,541 Bytes
8cf6d7a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """Two writer models on the same real محاور: pass rate, calls and cost.
The writer is the bill - 92% of a hundred-محور bag - and the model the
notebook tuned against is not the cheapest that writes this contract. But
cost is price times calls: a cheaper model the validator sends back twice as
often saves nothing. So before the setting changes, the two are run on the
same plans with the same sources, and the figures are read side by side.
python scripts/ab_writer.py --run <run id> --covers 10 \
--models anthropic/claude-haiku-4.5,google/gemini-2.5-flash
Spends tokens - about half a dollar for ten محاور on two models. Reads the
run's stored outputs; writes nothing to the database (the meter's flush loop
is not started, so no `llm_calls` rows are added either).
"""
from __future__ import annotations
import argparse
import asyncio
import sys
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from sqlalchemy import select # noqa: E402
from app.agents.arabic_bag.context import using # noqa: E402
from app.core.config import settings # noqa: E402
from app.db.models import Run, StepRun, StepStatus # noqa: E402
from app.db.session import SessionLocal # noqa: E402
from app.observability import llm_meter # noqa: E402
from app.observability.llm_meter import llm_context # noqa: E402
DEFAULT_MODELS = "anthropic/claude-haiku-4.5,google/gemini-2.5-flash"
async def _load(run_id: str | None) -> tuple[Run, dict[str, dict]]:
async with SessionLocal() as db:
if run_id:
run = await db.get(Run, uuid.UUID(run_id))
else:
run = await db.scalar(
select(Run).where(Run.pipeline_key == "arabic_training_bag",
Run.status == "succeeded")
.order_by(Run.queued_at.desc()).limit(1)
)
if run is None:
raise SystemExit("no such run")
rows = (await db.scalars(
select(StepRun).where(StepRun.run_id == run.id,
StepRun.status == StepStatus.succeeded)
)).all()
return run, {r.step_key: (r.output or {}) for r in rows}
def _spread(items: list, n: int) -> list:
"""`n` items spread over the list - every day and session represented."""
if n >= len(items):
return list(items)
step = len(items) / n
return [items[int(i * step)] for i in range(n)]
async def _write(model: str, blueprint: dict, plans: list, scraped: list,
credits: dict) -> dict:
from app.agents.arabic_bag import llms
from app.agents.arabic_bag.writer import write_one_cover
settings.bag_writer_model = model
llms._available = None # re-read the catalogue for the new name
tag = f"ab-{model.replace('/', '-')}"
llm_meter.seed_spent(tag, 0.0)
passed = calls = words = 0
started = time.time()
with llm_context(run_id=tag, step_run_id=None, user_id=None,
pipeline_key="ab_writer", step_key="bag_ar.content"):
with using(blueprint):
prev: list[str] = []
for i, plan in enumerate(plans, 1):
print(f"\n[{model}] {i}/{len(plans)} {str(plan['cover'])[:48]}")
res = await write_one_cover(plan, scraped, prev, credits=credits)
calls += int(res.get("_calls") or 0)
passed += 0 if res.get("_validation_failed") else 1
words += sum(len(str(p).split()) for b in res.get("blocks", [])
for p in (b.get("paragraphs") or []))
prev.append(f"{plan['cover']}: تم تناوله")
return {"model": model, "passed": passed, "calls": calls,
"usd": llm_meter.spent(tag), "seconds": time.time() - started,
"words": words}
async def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--run", help="run id; the latest finished bag when omitted")
parser.add_argument("--covers", type=int, default=10)
parser.add_argument("--models", default=DEFAULT_MODELS)
args = parser.parse_args()
llm_meter.install()
llm_meter.ensure()
run, outputs = await _load(args.run)
blueprint = outputs.get("bag_ar.blueprint") or {}
plans = (outputs.get("bag_ar.plan") or {}).get("plans") or []
scraped = (outputs.get("bag_ar.sources") or {}).get("value") or []
credits = (outputs.get("bag_ar.citations") or {}).get("value") or {}
if not plans:
raise SystemExit("the run has no plans to write from")
chosen = _spread(plans, args.covers)
print(f"run {run.id}: {len(chosen)} of {len(plans)} محاور, "
f"{len(args.models.split(','))} models")
results = []
for model in [m.strip() for m in args.models.split(",") if m.strip()]:
results.append(await _write(model, blueprint, chosen, scraped, credits))
print("\n" + "=" * 78)
print(f"{'model':36} {'pass':>5} {'calls':>6} {'$':>7} {'$/محور':>8} {'min':>5}")
for r in results:
n = len(chosen)
print(f"{r['model']:36} {r['passed']:>3}/{n:<2} {r['calls']:>6} "
f"{r['usd']:>7.3f} {r['usd'] / n:>8.4f} {r['seconds'] / 60:>5.1f}")
print("=" * 78)
print("القرار: الأرخص للمحور الواحد بنسبة نجاح مقاربة؛ الأرخص الذي يُرفض "
"ضعف الآخر لا يوفّر شيئًا.")
if __name__ == "__main__":
asyncio.run(main())
|