| """Build space/data.json for the side-by-side Space: base Dory vs Dory-Beam-4B vs Beam (the teacher) on the same |
| eval problems. |
| |
| .venv/bin/python -I scripts/build_space_data.py [--loops 2] [--seed 0] |
| |
| Problems are a seeded random sample per benchmark (not hand-picked); every answer is that model's first sample |
| for the problem from the eval runs. Also writes per-benchmark summary stats over the full eval. |
| |
| Beam's side ("teacher") is Beam-1.0-501B-A23B itself: self-hosted NVFP4, temperature 0.7, top-p 0.99, up to 28k |
| tokens, on the same 1,836 samples at each reasoning effort (data/beam_eval/, graded with verify.check as in cascade.py |
| and beam_effort_study.py). It follows the page's effort control via TEACHER_EFFORT: Dory-Beam low -> Beam low, |
| default (no effort tag) -> Beam medium (Beam's own default), high -> Beam high. |
| """ |
|
|
| import argparse |
| import json |
| import random |
| import sys |
| from collections import defaultdict |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(ROOT / "scripts")) |
| from verify import check |
|
|
| RES = ROOT / "data" / "eval" / "results" |
| |
| TEACHER = [ROOT / "data" / "beam_eval" / "run-evalbeam2-20261030-200821.jsonl", |
| ROOT / "data" / "beam_eval" / "run-evaleffort-20261101-183703.jsonl"] |
| |
| TEACHER_EFFORT = {"low": "low", "default": "medium", "high": "high"} |
| |
| QUOTA = {"math500": 10, "aime24": 5, "aime25": 5, "amc23": 5, "heldout": 9} |
| EFFORTS = ("low", "default", "high") |
| NAMES = {"gpqa": "GPQA-Diamond", "math500": "MATH-500", "aime24": "AIME 2024", "aime25": "AIME 2025", |
| "amc23": "AMC 2023", "heldout": "Held-out"} |
|
|
|
|
| def load(tag): |
| by = defaultdict(dict) |
| for line in open(RES / f"{tag}.jsonl"): |
| r = json.loads(line) |
| by[r["id"]].setdefault(r["sample"], r) |
| return by |
|
|
|
|
| def load_teacher(): |
| """Beam's answers per level, each in the same {id: {sample: record}} shape as load(); graded like |
| scripts/cascade.py and scripts/beam_effort_study.py. Only the levels the page shows are kept.""" |
| by = {lvl: defaultdict(dict) for lvl in TEACHER_EFFORT.values()} |
| for path in TEACHER: |
| for line in open(path): |
| r = json.loads(line) |
| if "error" in r or r["effort"] not in by: |
| continue |
| ok, pred = check(r["content"], r["gold"], r["answer_type"]) |
| by[r["effort"]][r["id"]].setdefault(r["sample_idx"], { |
| "bench": r["domain"], "tokens": r["usage"]["completion_tokens"], "pred": pred, |
| "correct": bool(ok and r["finish_reason"] == "stop"), "finish": r["finish_reason"], |
| "reasoning": (r["reasoning"] or "").strip(), "answer": (r["content"] or "").strip()}) |
| return by |
|
|
|
|
| def teacher_side(r): |
| return {"reasoning": r["reasoning"], "answer": r["answer"], "tokens": r["tokens"], "correct": r["correct"], |
| "pred": r["pred"], "truncated": r["finish"] == "length"} |
|
|
|
|
| def side(r): |
| reasoning, _, answer = r["text"].partition("</think>") |
| return {"reasoning": reasoning.strip(), "answer": answer.strip(), "tokens": r["tokens"], |
| "correct": r["correct"], "pred": r["pred"], "truncated": r["finish"] == "length"} |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--loops", type=int, default=2) |
| ap.add_argument("--seed", type=int, default=0) |
| args = ap.parse_args() |
| base = load(f"base_loops{args.loops}-64k") |
| |
| beam = {e: load(f"rE-t050_loops{args.loops}-{e}-64k") for e in EFFORTS} |
| teacher = load_teacher() |
| for lvl, runs in teacher.items(): |
| assert sum(map(len, runs.values())) == sum(map(len, base.values())), f"Beam {lvl} should cover every sample" |
| probs = [json.loads(l) for l in open(ROOT / "data" / "evalsets.jsonl")] |
| rng = random.Random(args.seed) |
|
|
| items = [] |
| for bench, k in QUOTA.items(): |
| pool = [p for p in probs if p["bench"] == bench] |
| for p in sorted(rng.sample(pool, k), key=lambda p: p["id"]): |
| items.append({"id": p["id"], "bench": NAMES[bench], "prompt": p["prompt"], "gold": p["answer"], |
| "base": side(base[p["id"]][0]), "beam": {e: side(beam[e][p["id"]][0]) for e in EFFORTS}, |
| "teacher": {lvl: teacher_side(runs[p["id"]][0]) for lvl, runs in teacher.items()}}) |
|
|
| stats = [] |
| for bench, name in NAMES.items(): |
| row = {"bench": name} |
| def agg(runs): |
| rs = [r for pid, samples in runs.items() for r in samples.values() if r["bench"] == bench] |
| return {"acc": round(100 * sum(r["correct"] for r in rs) / len(rs), 1), |
| "tokens": round(sum(r["tokens"] for r in rs) / len(rs))} |
| row["base"] = agg(base) |
| row["beam"] = {e: agg(beam[e]) for e in EFFORTS} |
| row["teacher"] = {lvl: agg(runs) for lvl, runs in teacher.items()} |
| stats.append(row) |
|
|
| out = ROOT / "space" / "data.json" |
| out.parent.mkdir(exist_ok=True) |
| out.write_text(json.dumps({"loops": args.loops, "seed": args.seed, "teacher_effort": TEACHER_EFFORT, |
| "stats": stats, "items": items}, ensure_ascii=False)) |
| both = sum(i["base"]["correct"] and i["beam"]["default"]["correct"] for i in items) |
| right = {lvl: sum(i["teacher"][lvl]["correct"] for i in items) for lvl in teacher} |
| print(f"{len(items)} problems ({both} both Dory sides correct; Beam correct {right}) -> {out} " |
| f"({out.stat().st_size / 1e6:.2f} MB)") |
| for lvl in teacher: |
| print(f" Beam {lvl:<6} " + " | ".join( |
| f"{s['bench']} {s['teacher'][lvl]['acc']:.1f} · {s['teacher'][lvl]['tokens'] / 1000:.1f}k" for s in stats)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
