hunterbown's picture
Include all 200 math questions and 400 full base/adapter responses
fa89a16 verified
Raw History Blame Contribute Delete
5.49 kB
{
"schema": "trainless-math-evaluation/1",
"scope": "Complete existing 200-item verification slice; requalification of exact analytic PEFT, not a newly held-out discovery",
"model": {
"base_model": "Qwen/Qwen3.5-2B",
"base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
"adapter_model_sha256": "f67ffbbc22c7a0c0e1dd9ce28459751b18ab91de6132390a164c611f5744e27b",
"adapter_config_sha256": "1cf3e3e143f8447c72287aa37419f1661b9afae6e8f7ba1bb848bed95a531752",
"dtype": "bfloat16"
},
"dataset": {
"id": "HuggingFaceH4/MATH-500",
"offset_zero_based": 64,
"last_index_inclusive": 263,
"n": 200,
"selected_item_sha256": "8a3b16a62f8b0755ebbd49d4fa3b3cd05a3dba3fa17e234f6001e92b3c36f6a1",
"selected_hash_algorithm": "sha256(json.dumps(items, sort_keys=True).encode()); default spaces and ensure_ascii=True; each item has only id, problem, answer",
"historical_hub_revision": null,
"historical_revision_recorded": false,
"current_upstream_verification": {
"verified_at": "2026-11-10T07:36:38Z",
"current_upstream_revision": "6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be",
"license_source_revision": "7ecc794703b2877f63226f2477a49b34f9b25163",
"upstream_test_jsonl_sha256": "35dc41080a3680858b27fa7e0533d2d547825316fc5dafe5d316f4ccc5a06132",
"current_upstream_selected_sha256": "8a3b16a62f8b0755ebbd49d4fa3b3cd05a3dba3fa17e234f6001e92b3c36f6a1"
}
},
"generation": {
"output_token_cap": 32768,
"max_num_seqs": 64,
"sampling": {
"temperature": 0.6,
"top_p": 0.95,
"top_k": 20,
"presence_penalty": 1.5,
"seed": 0,
"thinking": true
},
"runtime": {
"vllm": "0.31.0",
"torch": "2.13.0",
"transformers": "5.17.0",
"safetensors": "0.8.0",
"numpy": "2.2.6"
},
"gpu": "NVIDIA H100 PCIe"
},
"prompt_reconstruction": {
"method": "Reconstructed with the same pinned tokenizer and recorded historical Transformers version; original generation JSON did not store prompt strings",
"historical_prompts_stored": false,
"base_model": "Qwen/Qwen3.5-2B",
"base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
"transformers": "5.17.0",
"tokenizer_class": "Qwen2Tokenizer",
"tokenizer_config_sha256": "49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c",
"chat_template_sha256": "273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80",
"prompt_suffix": "Please reason step by step, and put your final answer within \\boxed{}.",
"thinking": true
},
"grading": {
"description": "repository evals.graders.for_benchmark(math500), unchanged",
"runtime": {
"python": "3.10.12",
"math-verify": "0.9.0",
"sympy": "1.12"
},
"source_sha256": {
"graders.py": "1cfec08ffbfaf9b95e25c87c2c4ea94847f0e47a149e279598df09d88ce1908c",
"benchmarks.py": "d69ac797dd546c6ab29dadfc516936199d1008f1271ea3514b6465db30619830",
"loops.py": "c93ab48e93ccfa18689d4b13a7dff3fb7a55389be528fb8576cb27b806e34e51"
}
},
"original_source_sha256": {
"generations.json": "868f8118474c7acf2361bfbbed29dc8c50d9813c395e269f4bad4e59854b02ff",
"graded.json": "d255d4ea60870bf9450cecff8ce909fce2b9b4a22fc94fbba76d03d48b7cf5c2",
"qualification-metadata.json": "3d334352fc4dba360b79174646a6f2d9fb259bfc79667f4909f768d28e9aacf4",
"reference_items_file": "c5198b451c6f1ca6bf4fcfbbac86161bd9a52442c6983fac60850a636b9eb680",
"prompt_reconstruction_file": "18162f85fd53db147cd810b84a91eb7ed4b35859717c1e35ca006368c1c9e2d8",
"attribution_file": "2a43117ca8354f96654ca990295ab1c490b6845a20a413223d3050dd1cb9bef4",
"original_benchmark_cache": "0baf4950c69ab971f1b45437b965927e5a2a8fda0d2c884c408c9b9c087fca84",
"generation_driver": "4cbe1f202744ece0c57aa2b7e4d5e3aa1235fa27e2096595aedccc5996a221ff"
},
"results": {
"base": {
"n": 200,
"correct": 136,
"accuracy": 0.68,
"wilson95": [
0.6124810156588646,
0.7407346668453195
],
"capped": 63,
"total_output_tokens": 3263048,
"generation_wall_seconds": 759.5006894249236
},
"adapter": {
"n": 200,
"correct": 152,
"accuracy": 0.76,
"wilson95": [
0.6962665902079583,
0.8139338400758631
],
"capped": 48,
"total_output_tokens": 2863410,
"generation_wall_seconds": 760.9235217790119
}
},
"paired": {
"accuracy_delta": 0.08,
"bootstrap95": [
0.03,
0.135
],
"bootstrap_repetitions": 20000,
"seed": 20261110,
"base_wrong_adapter_right": 23,
"base_right_adapter_wrong": 7,
"note": "paired item bootstrap, same saved questions; model sampling variance from additional seeds unmeasured"
},
"limitations": [
"Same previously evaluated questions; no new held-out discovery claim.",
"One sampling seed; seed variance unmeasured.",
"No wall-clock speedup established; both generation arms took about 760 seconds.",
"Only the exact analytic adapter/runtime is measured; merged and GGUF variants do not inherit the score.",
"Historical Hub dataset revision was not saved. Current upstream identity was verified separately.",
"Prompt strings were reconstructed from the pinned tokenizer and original runtime, not recovered from the generation JSON.",
"Automatic answer labels can miss contradictions; full outputs are included for human review."
]
}