File size: 7,540 Bytes
f9d4a1a b1cf75e f9d4a1a 079e13c 0052e99 d33ce47 f9d4a1a 079e13c f9d4a1a 079e13c f9d4a1a d33ce47 0052e99 f9d4a1a 079e13c 0052e99 d33ce47 f9d4a1a 0052e99 f9d4a1a d33ce47 f9d4a1a 0052e99 f9d4a1a 079e13c f9d4a1a d33ce47 f9d4a1a b1cf75e f9d4a1a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """Export every scored eval answer into release/dataset/eval/ (Parquet) so the results tables can be audited.
.venv/bin/python -I scripts/export_eval_data.py
eval/benchmarks/<model>_loops<L>[_efficient].parquet one row per sampled answer (GPQA, MATH-500, AIME, AMC, held-out)
eval/ifeval/<model>_loops2.parquet one row per IFEval answer
eval/problems.parquet prompt + reference answer per problem, except GPQA
Models: beam-1.0-501b-a23b (the teacher: self-hosted NVFP4 on SGLang, temperature 0.7, top-p 0.99, up to 28k
tokens, one file per reasoning effort; loops is 0), dory1.0-4b (base), dory-beam-4b (v2, with reasoning_effort; `reasoning_mode` effort-low/medium/high or
normal for the untagged default), dory-beam-4b-v1 (the first release), dory-beam-4b-accurate, and ablations.
Base Dory and v1 were run with a 32k limit (16k for IFEval) and every answer that hit it was resumed to 64k by
scripts/spawn_continue.py (`resumed_tokens` > 0). v2 and Accurate ran with 64k from the start. The ablations rarely hit
32k and stay there; `max_output_tokens` says which limit applies.
GPQA's authors ask that its questions not appear online in plain text (to keep them out of training data), so GPQA
rows keep their grades, extracted letters and token counts but drop the question and the model's text.
"""
import glob
import json
import sys
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
ROOT = Path(__file__).resolve().parent.parent
RES = ROOT / "data" / "eval" / "results"
OUT = ROOT / "release" / "dataset" / "eval"
# local result file -> (model name in the dataset, loops, reasoning mode)
RUNS = {
"base_loops1-64k": ("dory1.0-4b", 1, "normal"), "base_loops2-64k": ("dory1.0-4b", 2, "normal"),
"base_loops4-64k": ("dory1.0-4b", 4, "normal"), "base_loops2eff": ("dory1.0-4b", 2, "efficient"),
"mix050_loops1-64k": ("dory-beam-4b-v1", 1, "normal"), "mix050_loops2-64k": ("dory-beam-4b-v1", 2, "normal"),
"mix050_loops4-64k": ("dory-beam-4b-v1", 4, "normal"),
"rE-t050_loops2-default-64k": ("dory-beam-4b", 2, "normal"),
"rE-t050_loops2-low-64k": ("dory-beam-4b", 2, "effort-low"),
"rE-t050_loops2-medium-64k": ("dory-beam-4b", 2, "effort-medium"),
"rE-t050_loops2-high-64k": ("dory-beam-4b", 2, "effort-high"),
"rE-t050_loops4-high-64k": ("dory-beam-4b", 4, "effort-high"),
"mix030_loops2-64k": ("dory-beam-4b-accurate", 2, "normal"),
"mix030_loops4-64k": ("dory-beam-4b-accurate", 4, "normal"),
"mix070_loops2": ("blend-70-30", 2, "normal"),
"rA_loops1": ("run-a-sft", 1, "normal"), "rA_loops2": ("run-a-sft", 2, "normal"),
"rA_loops4": ("run-a-sft", 4, "normal"),
"rA400_loops2": ("run-a-step400", 2, "normal"), "rA800_loops2": ("run-a-step800", 2, "normal"),
"rA2_loops1": ("run-a2-random-trace", 1, "normal"), "rA2_loops2": ("run-a2-random-trace", 2, "normal"),
"rA2_loops4": ("run-a2-random-trace", 4, "normal"),
}
# Beam answered the same samples: data/beam_eval/run-evalbeam2-* (medium) and run-evaleffort-* (other levels)
BEAM_FILES = ["run-evalbeam2-*.jsonl", "run-evaleffort-*.jsonl"]
IFEVAL = {"base_ifeval_loops2-64k": "dory1.0-4b", "mix050_ifeval_loops2-64k": "dory-beam-4b-v1",
"rE-t050_ifeval_loops2-64k": "dory-beam-4b"}
def limit(tag):
return 65536 if tag.endswith("-64k") else 16384 if "ifeval" in tag else 32768
def main():
problems = {json.loads(l)["id"]: json.loads(l) for l in open(ROOT / "data" / "evalsets.jsonl")}
for f in list((OUT / "benchmarks").glob("*.parquet")) + list((OUT / "ifeval").glob("*.parquet")):
f.unlink() # file names changed between releases; start clean
(OUT / "benchmarks").mkdir(parents=True, exist_ok=True)
(OUT / "ifeval").mkdir(parents=True, exist_ok=True)
for tag, (model, loops, mode) in RUNS.items():
rows = []
for line in open(RES / f"{tag}.jsonl"):
r = json.loads(line)
gpqa = r["bench"] == "gpqa"
rows.append({"model": model, "loops": loops, "reasoning_mode": mode, "bench": r["bench"],
"problem_id": r["id"], "sample": r["sample"], "correct": r["correct"],
"extracted_answer": r["pred"], "reference_answer": problems[r["id"]]["answer"],
"output_tokens": r["tokens"], "finish_reason": r["finish"],
"max_output_tokens": limit(tag), "resumed_tokens": r.get("continued_tokens", 0),
"thinking_closed": r["thinking_closed"], "text": None if gpqa else r["text"]})
name = f"{model}_loops{loops}{'' if mode == 'normal' else '_' + mode}.parquet"
pq.write_table(pa.Table.from_pylist(rows), OUT / "benchmarks" / name, compression="zstd")
print(f"{name}: {len(rows)} rows")
sys.path.insert(0, str(ROOT / "scripts"))
from verify import check
beam = {}
for pattern in BEAM_FILES:
for path in sorted(glob.glob(str(ROOT / "data" / "beam_eval" / pattern))):
for line in open(path):
r = json.loads(line)
if "error" in r:
continue
p = problems[r["id"]]
gpqa = p["bench"] == "gpqa"
ok, pred = check(r["content"], r["gold"], r["answer_type"])
stop = r["finish_reason"] == "stop"
beam.setdefault(r["effort"], []).append({
"model": "beam-1.0-501b-a23b", "loops": 0, "reasoning_mode": f"effort-{r['effort']}",
"bench": p["bench"], "problem_id": r["id"], "sample": r["sample_idx"], "correct": bool(ok and stop),
"extracted_answer": pred, "reference_answer": p["answer"],
"output_tokens": r["usage"]["completion_tokens"], "finish_reason": r["finish_reason"],
"max_output_tokens": 28000, "resumed_tokens": 0, "thinking_closed": stop,
"text": None if gpqa else r["reasoning"] + "</think>" + r["content"]})
for eff, rows in beam.items():
name = f"beam-1.0-501b-a23b_effort-{eff}.parquet"
pq.write_table(pa.Table.from_pylist(rows), OUT / "benchmarks" / name, compression="zstd")
print(f"{name}: {len(rows)} rows")
for tag, model in IFEVAL.items():
rows = []
for line in open(RES / f"{tag}.jsonl"):
r = json.loads(line)
rows.append({"model": model, "loops": 2, "key": r["key"], "sample": r["sample"],
"strict_prompt": r["strict_prompt"], "strict_instructions": r["strict_inst"],
"loose_prompt": r["loose_prompt"], "loose_instructions": r["loose_inst"],
"output_tokens": r["tokens"], "finish_reason": r["finish"],
"max_output_tokens": limit(tag), "resumed_tokens": r.get("continued_tokens", 0),
"text": r["text"]})
pq.write_table(pa.Table.from_pylist(rows), OUT / "ifeval" / f"{model}_loops2.parquet", compression="zstd")
print(f"ifeval/{model}_loops2.parquet: {len(rows)} rows")
rows = [{"bench": p["bench"], "problem_id": p["id"], "prompt": p["prompt"], "reference_answer": p["answer"],
"answer_type": p["answer_type"], "samples": p["samples"]} for p in problems.values() if p["bench"] != "gpqa"]
pq.write_table(pa.Table.from_pylist(rows), OUT / "problems.parquet", compression="zstd")
print(f"problems.parquet: {len(rows)} rows")
if __name__ == "__main__":
main()
|