Blame for code/scripts/build_space_data.py

"""Build space/data.json for the side-by-side Space: base Dory vs Dory-Beam-4B vs Beam (the teacher) on the same
eval problems.

  .venv/bin/python -I scripts/build_space_data.py [--loops 2] [--seed 0]

Problems are a seeded random sample per benchmark (not hand-picked); every answer is that model's first sample
for the problem from the eval runs. Also writes per-benchmark summary stats over the full eval.

Beam's side ("teacher") is Beam-1.0-501B-A23B itself: self-hosted NVFP4, temperature 0.7, top-p 0.99, up to 28k
tokens, on the same 1,836 samples at each reasoning effort (data/beam_eval/, graded with verify.check as in cascade.py
and beam_effort_study.py). It follows the page's effort control via TEACHER_EFFORT: Dory-Beam low -> Beam low,
default (no effort tag) -> Beam medium (Beam's own default), high -> Beam high.
"""

import argparse
import json
import random
import sys
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from verify import check  # noqa: E402

RES = ROOT / "data" / "eval" / "results"
# Beam at medium, then at every other level (minimal, low, high, xhigh, max)
TEACHER = [ROOT / "data" / "beam_eval" / "run-evalbeam2-20261030-200821.jsonl",
           ROOT / "data" / "beam_eval" / "run-evaleffort-20261101-183703.jsonl"]
# Dory-Beam effort on the page -> the Beam level shown beside it
TEACHER_EFFORT = {"low": "low", "default": "medium", "high": "high"}
# No GPQA examples: its authors ask that questions not appear online in plain text. GPQA stays in the stats table.
QUOTA = {"math500": 10, "aime24": 5, "aime25": 5, "amc23": 5, "heldout": 9}
EFFORTS = ("low", "default", "high")
NAMES = {"gpqa": "GPQA-Diamond", "math500": "MATH-500", "aime24": "AIME 2024", "aime25": "AIME 2025",
         "amc23": "AMC 2023", "heldout": "Held-out"}


def load(tag):
    by = defaultdict(dict)
    for line in open(RES / f"{tag}.jsonl"):
        r = json.loads(line)
        by[r["id"]].setdefault(r["sample"], r)
    return by


def load_teacher():
    """Beam's answers per level, each in the same {id: {sample: record}} shape as load(); graded like
    scripts/cascade.py and scripts/beam_effort_study.py. Only the levels the page shows are kept."""
    by = {lvl: defaultdict(dict) for lvl in TEACHER_EFFORT.values()}
    for path in TEACHER:
        for line in open(path):
            r = json.loads(line)
            if "error" in r or r["effort"] not in by:
                continue
            ok, pred = check(r["content"], r["gold"], r["answer_type"])
            by[r["effort"]][r["id"]].setdefault(r["sample_idx"], {
                "bench": r["domain"], "tokens": r["usage"]["completion_tokens"], "pred": pred,
                "correct": bool(ok and r["finish_reason"] == "stop"), "finish": r["finish_reason"],
                "reasoning": (r["reasoning"] or "").strip(), "answer": (r["content"] or "").strip()})
    return by


def teacher_side(r):
    return {"reasoning": r["reasoning"], "answer": r["answer"], "tokens": r["tokens"], "correct": r["correct"],
            "pred": r["pred"], "truncated": r["finish"] == "length"}


def side(r):
    reasoning, _, answer = r["text"].partition("</think>")
    return {"reasoning": reasoning.strip(), "answer": answer.strip(), "tokens": r["tokens"],
            "correct": r["correct"], "pred": r["pred"], "truncated": r["finish"] == "length"}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--loops", type=int, default=2)
    ap.add_argument("--seed", type=int, default=0)
    args = ap.parse_args()
    base = load(f"base_loops{args.loops}-64k")
    # Dory-Beam-4B v2 at each reasoning effort ("default" = no effort tag)
    beam = {e: load(f"rE-t050_loops{args.loops}-{e}-64k") for e in EFFORTS}
    teacher = load_teacher()
    for lvl, runs in teacher.items():
        assert sum(map(len, runs.values())) == sum(map(len, base.values())), f"Beam {lvl} should cover every sample"
    probs = [json.loads(l) for l in open(ROOT / "data" / "evalsets.jsonl")]
    rng = random.Random(args.seed)

    items = []
    for bench, k in QUOTA.items():
        pool = [p for p in probs if p["bench"] == bench]
        for p in sorted(rng.sample(pool, k), key=lambda p: p["id"]):
            items.append({"id": p["id"], "bench": NAMES[bench], "prompt": p["prompt"], "gold": p["answer"],
                          "base": side(base[p["id"]][0]), "beam": {e: side(beam[e][p["id"]][0]) for e in EFFORTS},
                          "teacher": {lvl: teacher_side(runs[p["id"]][0]) for lvl, runs in teacher.items()}})

    stats = []
    for bench, name in NAMES.items():
        row = {"bench": name}
        def agg(runs):
            rs = [r for pid, samples in runs.items() for r in samples.values() if r["bench"] == bench]
            return {"acc": round(100 * sum(r["correct"] for r in rs) / len(rs), 1),
                    "tokens": round(sum(r["tokens"] for r in rs) / len(rs))}
        row["base"] = agg(base)
        row["beam"] = {e: agg(beam[e]) for e in EFFORTS}
        row["teacher"] = {lvl: agg(runs) for lvl, runs in teacher.items()}
        stats.append(row)

    out = ROOT / "space" / "data.json"
    out.parent.mkdir(exist_ok=True)
    out.write_text(json.dumps({"loops": args.loops, "seed": args.seed, "teacher_effort": TEACHER_EFFORT,
                               "stats": stats, "items": items}, ensure_ascii=False))
    both = sum(i["base"]["correct"] and i["beam"]["default"]["correct"] for i in items)
    right = {lvl: sum(i["teacher"][lvl]["correct"] for i in items) for lvl in teacher}
    print(f"{len(items)} problems ({both} both Dory sides correct; Beam correct {right}) -> {out} "
          f"({out.stat().st_size / 1e6:.2f} MB)")
    for lvl in teacher:
        print(f"  Beam {lvl:<6} " + " | ".join(
            f"{s['bench']} {s['teacher'][lvl]['acc']:.1f} · {s['teacher'][lvl]['tokens'] / 1000:.1f}k" for s in stats))


if __name__ == "__main__":
    main()