File size: 5,492 Bytes
fa89a16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
{
  "schema": "trainless-math-evaluation/1",
  "scope": "Complete existing 200-item verification slice; requalification of exact analytic PEFT, not a newly held-out discovery",
  "model": {
    "base_model": "Qwen/Qwen3.5-2B",
    "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
    "adapter_model_sha256": "f67ffbbc22c7a0c0e1dd9ce28459751b18ab91de6132390a164c611f5744e27b",
    "adapter_config_sha256": "1cf3e3e143f8447c72287aa37419f1661b9afae6e8f7ba1bb848bed95a531752",
    "dtype": "bfloat16"
  },
  "dataset": {
    "id": "HuggingFaceH4/MATH-500",
    "offset_zero_based": 64,
    "last_index_inclusive": 263,
    "n": 200,
    "selected_item_sha256": "8a3b16a62f8b0755ebbd49d4fa3b3cd05a3dba3fa17e234f6001e92b3c36f6a1",
    "selected_hash_algorithm": "sha256(json.dumps(items, sort_keys=True).encode()); default spaces and ensure_ascii=True; each item has only id, problem, answer",
    "historical_hub_revision": null,
    "historical_revision_recorded": false,
    "current_upstream_verification": {
      "verified_at": "2026-11-10T07:36:38Z",
      "current_upstream_revision": "6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be",
      "license_source_revision": "7ecc794703b2877f63226f2477a49b34f9b25163",
      "upstream_test_jsonl_sha256": "35dc41080a3680858b27fa7e0533d2d547825316fc5dafe5d316f4ccc5a06132",
      "current_upstream_selected_sha256": "8a3b16a62f8b0755ebbd49d4fa3b3cd05a3dba3fa17e234f6001e92b3c36f6a1"
    }
  },
  "generation": {
    "output_token_cap": 32768,
    "max_num_seqs": 64,
    "sampling": {
      "temperature": 0.6,
      "top_p": 0.95,
      "top_k": 20,
      "presence_penalty": 1.5,
      "seed": 0,
      "thinking": true
    },
    "runtime": {
      "vllm": "0.31.0",
      "torch": "2.13.0",
      "transformers": "5.17.0",
      "safetensors": "0.8.0",
      "numpy": "2.2.6"
    },
    "gpu": "NVIDIA H100 PCIe"
  },
  "prompt_reconstruction": {
    "method": "Reconstructed with the same pinned tokenizer and recorded historical Transformers version; original generation JSON did not store prompt strings",
    "historical_prompts_stored": false,
    "base_model": "Qwen/Qwen3.5-2B",
    "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
    "transformers": "5.17.0",
    "tokenizer_class": "Qwen2Tokenizer",
    "tokenizer_config_sha256": "49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c",
    "chat_template_sha256": "273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80",
    "prompt_suffix": "Please reason step by step, and put your final answer within \\boxed{}.",
    "thinking": true
  },
  "grading": {
    "description": "repository evals.graders.for_benchmark(math500), unchanged",
    "runtime": {
      "python": "3.10.12",
      "math-verify": "0.9.0",
      "sympy": "1.12"
    },
    "source_sha256": {
      "graders.py": "1cfec08ffbfaf9b95e25c87c2c4ea94847f0e47a149e279598df09d88ce1908c",
      "benchmarks.py": "d69ac797dd546c6ab29dadfc516936199d1008f1271ea3514b6465db30619830",
      "loops.py": "c93ab48e93ccfa18689d4b13a7dff3fb7a55389be528fb8576cb27b806e34e51"
    }
  },
  "original_source_sha256": {
    "generations.json": "868f8118474c7acf2361bfbbed29dc8c50d9813c395e269f4bad4e59854b02ff",
    "graded.json": "d255d4ea60870bf9450cecff8ce909fce2b9b4a22fc94fbba76d03d48b7cf5c2",
    "qualification-metadata.json": "3d334352fc4dba360b79174646a6f2d9fb259bfc79667f4909f768d28e9aacf4",
    "reference_items_file": "c5198b451c6f1ca6bf4fcfbbac86161bd9a52442c6983fac60850a636b9eb680",
    "prompt_reconstruction_file": "18162f85fd53db147cd810b84a91eb7ed4b35859717c1e35ca006368c1c9e2d8",
    "attribution_file": "2a43117ca8354f96654ca990295ab1c490b6845a20a413223d3050dd1cb9bef4",
    "original_benchmark_cache": "0baf4950c69ab971f1b45437b965927e5a2a8fda0d2c884c408c9b9c087fca84",
    "generation_driver": "4cbe1f202744ece0c57aa2b7e4d5e3aa1235fa27e2096595aedccc5996a221ff"
  },
  "results": {
    "base": {
      "n": 200,
      "correct": 136,
      "accuracy": 0.68,
      "wilson95": [
        0.6124810156588646,
        0.7407346668453195
      ],
      "capped": 63,
      "total_output_tokens": 3263048,
      "generation_wall_seconds": 759.5006894249236
    },
    "adapter": {
      "n": 200,
      "correct": 152,
      "accuracy": 0.76,
      "wilson95": [
        0.6962665902079583,
        0.8139338400758631
      ],
      "capped": 48,
      "total_output_tokens": 2863410,
      "generation_wall_seconds": 760.9235217790119
    }
  },
  "paired": {
    "accuracy_delta": 0.08,
    "bootstrap95": [
      0.03,
      0.135
    ],
    "bootstrap_repetitions": 20000,
    "seed": 20261110,
    "base_wrong_adapter_right": 23,
    "base_right_adapter_wrong": 7,
    "note": "paired item bootstrap, same saved questions; model sampling variance from additional seeds unmeasured"
  },
  "limitations": [
    "Same previously evaluated questions; no new held-out discovery claim.",
    "One sampling seed; seed variance unmeasured.",
    "No wall-clock speedup established; both generation arms took about 760 seconds.",
    "Only the exact analytic adapter/runtime is measured; merged and GGUF variants do not inherit the score.",
    "Historical Hub dataset revision was not saved. Current upstream identity was verified separately.",
    "Prompt strings were reconstructed from the pinned tokenizer and original runtime, not recovered from the generation JSON.",
    "Automatic answer labels can miss contradictions; full outputs are included for human review."
  ]
}