File size: 5,541 Bytes
8cf6d7a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
"""Two writer models on the same real محاور: pass rate, calls and cost.

The writer is the bill - 92% of a hundred-محور bag - and the model the
notebook tuned against is not the cheapest that writes this contract. But
cost is price times calls: a cheaper model the validator sends back twice as
often saves nothing. So before the setting changes, the two are run on the
same plans with the same sources, and the figures are read side by side.

    python scripts/ab_writer.py --run <run id> --covers 10 \
        --models anthropic/claude-haiku-4.5,google/gemini-2.5-flash

Spends tokens - about half a dollar for ten محاور on two models. Reads the
run's stored outputs; writes nothing to the database (the meter's flush loop
is not started, so no `llm_calls` rows are added either).
"""

from __future__ import annotations

import argparse
import asyncio
import sys
import time
import uuid
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from sqlalchemy import select  # noqa: E402

from app.agents.arabic_bag.context import using  # noqa: E402
from app.core.config import settings  # noqa: E402
from app.db.models import Run, StepRun, StepStatus  # noqa: E402
from app.db.session import SessionLocal  # noqa: E402
from app.observability import llm_meter  # noqa: E402
from app.observability.llm_meter import llm_context  # noqa: E402

DEFAULT_MODELS = "anthropic/claude-haiku-4.5,google/gemini-2.5-flash"


async def _load(run_id: str | None) -> tuple[Run, dict[str, dict]]:
    async with SessionLocal() as db:
        if run_id:
            run = await db.get(Run, uuid.UUID(run_id))
        else:
            run = await db.scalar(
                select(Run).where(Run.pipeline_key == "arabic_training_bag",
                                  Run.status == "succeeded")
                .order_by(Run.queued_at.desc()).limit(1)
            )
        if run is None:
            raise SystemExit("no such run")
        rows = (await db.scalars(
            select(StepRun).where(StepRun.run_id == run.id,
                                  StepRun.status == StepStatus.succeeded)
        )).all()
        return run, {r.step_key: (r.output or {}) for r in rows}


def _spread(items: list, n: int) -> list:
    """`n` items spread over the list - every day and session represented."""
    if n >= len(items):
        return list(items)
    step = len(items) / n
    return [items[int(i * step)] for i in range(n)]


async def _write(model: str, blueprint: dict, plans: list, scraped: list,
                 credits: dict) -> dict:
    from app.agents.arabic_bag import llms
    from app.agents.arabic_bag.writer import write_one_cover

    settings.bag_writer_model = model
    llms._available = None  # re-read the catalogue for the new name
    tag = f"ab-{model.replace('/', '-')}"
    llm_meter.seed_spent(tag, 0.0)
    passed = calls = words = 0
    started = time.time()
    with llm_context(run_id=tag, step_run_id=None, user_id=None,
                     pipeline_key="ab_writer", step_key="bag_ar.content"):
        with using(blueprint):
            prev: list[str] = []
            for i, plan in enumerate(plans, 1):
                print(f"\n[{model}] {i}/{len(plans)} {str(plan['cover'])[:48]}")
                res = await write_one_cover(plan, scraped, prev, credits=credits)
                calls += int(res.get("_calls") or 0)
                passed += 0 if res.get("_validation_failed") else 1
                words += sum(len(str(p).split()) for b in res.get("blocks", [])
                             for p in (b.get("paragraphs") or []))
                prev.append(f"{plan['cover']}: تم تناوله")
    return {"model": model, "passed": passed, "calls": calls,
            "usd": llm_meter.spent(tag), "seconds": time.time() - started,
            "words": words}


async def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    parser.add_argument("--run", help="run id; the latest finished bag when omitted")
    parser.add_argument("--covers", type=int, default=10)
    parser.add_argument("--models", default=DEFAULT_MODELS)
    args = parser.parse_args()

    llm_meter.install()
    llm_meter.ensure()
    run, outputs = await _load(args.run)
    blueprint = outputs.get("bag_ar.blueprint") or {}
    plans = (outputs.get("bag_ar.plan") or {}).get("plans") or []
    scraped = (outputs.get("bag_ar.sources") or {}).get("value") or []
    credits = (outputs.get("bag_ar.citations") or {}).get("value") or {}
    if not plans:
        raise SystemExit("the run has no plans to write from")
    chosen = _spread(plans, args.covers)
    print(f"run {run.id}: {len(chosen)} of {len(plans)} محاور, "
          f"{len(args.models.split(','))} models")

    results = []
    for model in [m.strip() for m in args.models.split(",") if m.strip()]:
        results.append(await _write(model, blueprint, chosen, scraped, credits))

    print("\n" + "=" * 78)
    print(f"{'model':36} {'pass':>5} {'calls':>6} {'$':>7} {'$/محور':>8} {'min':>5}")
    for r in results:
        n = len(chosen)
        print(f"{r['model']:36} {r['passed']:>3}/{n:<2} {r['calls']:>6} "
              f"{r['usd']:>7.3f} {r['usd'] / n:>8.4f} {r['seconds'] / 60:>5.1f}")
    print("=" * 78)
    print("القرار: الأرخص للمحور الواحد بنسبة نجاح مقاربة؛ الأرخص الذي يُرفض "
          "ضعف الآخر لا يوفّر شيئًا.")


if __name__ == "__main__":
    asyncio.run(main())