reed-meyerson commited on
Commit
5fa10ce
·
verified ·
1 Parent(s): cde9a98

Upload folder using huggingface_hub

Browse files
bf16_every_eval_ever/aime25.json ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "aime25|0/CohereLabs/command-a-plus-05-2026-bf16/1783690408.011839",
4
+ "evaluation_timestamp": "777717",
5
+ "retrieved_timestamp": "1783690408.011839",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "0.13.1.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "provider": "hosted_vllm",
26
+ "base_url": "http://localhost:8000/v1",
27
+ "concurrent_requests": "16",
28
+ "verbose": "False",
29
+ "max_model_length": "131072",
30
+ "api_max_retry": "8",
31
+ "api_retry_sleep": "1.0",
32
+ "api_retry_multiplier": "2.0",
33
+ "timeout": "3600.0",
34
+ "num_seeds_merged": "8"
35
+ }
36
+ },
37
+ "evaluation_results": [
38
+ {
39
+ "evaluation_name": "aime25",
40
+ "source_data": {
41
+ "dataset_name": "aime25",
42
+ "source_type": "hf_dataset",
43
+ "hf_repo": "yentinglin/aime_2025",
44
+ "hf_split": "train"
45
+ },
46
+ "metric_config": {
47
+ "evaluation_description": "pass@k:k=1&n=1",
48
+ "lower_is_better": false,
49
+ "score_type": "continuous",
50
+ "min_score": 0.0,
51
+ "max_score": 1.0
52
+ },
53
+ "score_details": {
54
+ "score": 0.9041666666666667,
55
+ "details": {
56
+ "seed_scores": "[0.8666666666666667, 0.9, 0.9333333333333333, 0.8333333333333334, 0.9333333333333333, 0.8666666666666667, 0.9333333333333333, 0.9666666666666667]",
57
+ "evaluation_timestamps": "[782779, 797599, 793930, 787744, 777717, 795167, 792988, 796501]",
58
+ "seed_values": "[1, 7, 4, 2, 0, 5, 3, 6]"
59
+ },
60
+ "uncertainty": {
61
+ "standard_error": {
62
+ "value": 0.015983001883449463,
63
+ "method": "across_seeds"
64
+ },
65
+ "num_samples": 8
66
+ }
67
+ },
68
+ "generation_config": {
69
+ "generation_args": {
70
+ "temperature": 0.6,
71
+ "top_p": 0.95,
72
+ "max_tokens": 65536,
73
+ "max_attempts": 1
74
+ },
75
+ "additional_details": {
76
+ "seed": "1",
77
+ "num_fewshot": "0"
78
+ }
79
+ }
80
+ },
81
+ {
82
+ "evaluation_name": "aime25",
83
+ "source_data": {
84
+ "dataset_name": "aime25",
85
+ "source_type": "hf_dataset",
86
+ "hf_repo": "yentinglin/aime_2025",
87
+ "hf_split": "train"
88
+ },
89
+ "metric_config": {
90
+ "evaluation_description": "avg@n:n=1",
91
+ "lower_is_better": false,
92
+ "score_type": "continuous",
93
+ "min_score": 0.0,
94
+ "max_score": 1.0
95
+ },
96
+ "score_details": {
97
+ "score": 0.9041666666666667,
98
+ "details": {
99
+ "seed_scores": "[0.8666666666666667, 0.9, 0.9333333333333333, 0.8333333333333334, 0.9333333333333333, 0.8666666666666667, 0.9333333333333333, 0.9666666666666667]",
100
+ "evaluation_timestamps": "[782779, 797599, 793930, 787744, 777717, 795167, 792988, 796501]",
101
+ "seed_values": "[1, 7, 4, 2, 0, 5, 3, 6]"
102
+ },
103
+ "uncertainty": {
104
+ "standard_error": {
105
+ "value": 0.015983001883449463,
106
+ "method": "across_seeds"
107
+ },
108
+ "num_samples": 8
109
+ }
110
+ },
111
+ "generation_config": {
112
+ "generation_args": {
113
+ "temperature": 0.6,
114
+ "top_p": 0.95,
115
+ "max_tokens": 65536,
116
+ "max_attempts": 1
117
+ },
118
+ "additional_details": {
119
+ "seed": "1",
120
+ "num_fewshot": "0"
121
+ }
122
+ }
123
+ }
124
+ ]
125
+ }
bf16_every_eval_ever/gpqa_diamond.json ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "gpqa:diamond|0/CohereLabs/command-a-plus-05-2026-bf16/1783690408.011196",
4
+ "evaluation_timestamp": "777717",
5
+ "retrieved_timestamp": "1783690408.011196",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "0.13.1.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "provider": "hosted_vllm",
26
+ "base_url": "http://localhost:8000/v1",
27
+ "concurrent_requests": "16",
28
+ "verbose": "False",
29
+ "max_model_length": "131072",
30
+ "api_max_retry": "8",
31
+ "api_retry_sleep": "1.0",
32
+ "api_retry_multiplier": "2.0",
33
+ "timeout": "3600.0",
34
+ "num_seeds_merged": "3"
35
+ }
36
+ },
37
+ "evaluation_results": [
38
+ {
39
+ "evaluation_name": "gpqa:diamond",
40
+ "source_data": {
41
+ "dataset_name": "gpqa:diamond",
42
+ "source_type": "hf_dataset",
43
+ "hf_repo": "Idavidrein/gpqa",
44
+ "hf_split": "train",
45
+ "additional_details": {
46
+ "hf_subset": "gpqa_diamond"
47
+ }
48
+ },
49
+ "metric_config": {
50
+ "evaluation_description": "gpqa_pass@k:k=1",
51
+ "lower_is_better": false,
52
+ "score_type": "continuous",
53
+ "min_score": 0.0,
54
+ "max_score": 1.0
55
+ },
56
+ "score_details": {
57
+ "score": 0.744107744107744,
58
+ "details": {
59
+ "seed_scores": "[0.7474747474747475, 0.7676767676767676, 0.7171717171717171]",
60
+ "evaluation_timestamps": "[777717, 782779, 787744]",
61
+ "seed_values": "[0, 1, 2]"
62
+ },
63
+ "uncertainty": {
64
+ "standard_error": {
65
+ "value": 0.014676427419328869,
66
+ "method": "across_seeds"
67
+ },
68
+ "num_samples": 3
69
+ }
70
+ },
71
+ "generation_config": {
72
+ "generation_args": {
73
+ "temperature": 0.6,
74
+ "top_p": 0.95,
75
+ "max_tokens": 65536,
76
+ "max_attempts": 1
77
+ },
78
+ "additional_details": {
79
+ "seed": "0",
80
+ "num_fewshot": "0"
81
+ }
82
+ }
83
+ }
84
+ ]
85
+ }
bf16_every_eval_ever/gsm8k_platinum_cot_llama.json ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "gsm8k_platinum_cot_llama/CohereLabs/command-a-plus-05-2026-bf16/1783690381.879346",
4
+ "evaluation_timestamp": "1782830712",
5
+ "retrieved_timestamp": "1783690381.879346",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "model_args": "{'model': 'CohereLabs/command-a-plus-05-2026-bf16', 'max_length': 72000, 'base_url': 'http://localhost:8000/v1/chat/completions', 'num_concurrent': 128, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 3600}",
26
+ "num_seeds_merged": "3"
27
+ }
28
+ },
29
+ "evaluation_results": [
30
+ {
31
+ "evaluation_name": "gsm8k_platinum_cot_llama/strict-match",
32
+ "source_data": {
33
+ "dataset_name": "gsm8k_platinum_cot_llama",
34
+ "source_type": "hf_dataset",
35
+ "hf_repo": "madrylab/gsm8k-platinum",
36
+ "hf_split": "test"
37
+ },
38
+ "metric_config": {
39
+ "evaluation_description": "exact_match (filter: strict-match)",
40
+ "lower_is_better": false,
41
+ "score_type": "continuous",
42
+ "min_score": 0.0,
43
+ "max_score": 1.0
44
+ },
45
+ "score_details": {
46
+ "score": 0.9498207885304659,
47
+ "details": {
48
+ "seed_scores": "[0.9520264681555004, 0.9487179487179487, 0.9487179487179487]",
49
+ "evaluation_timestamps": "[1782872444, 1782830712, 1782851542]",
50
+ "seed_values": "[2, 0, 1]"
51
+ },
52
+ "uncertainty": {
53
+ "standard_error": {
54
+ "value": 0.0011028398125172423,
55
+ "method": "across_seeds"
56
+ },
57
+ "num_samples": 3
58
+ }
59
+ },
60
+ "generation_config": {
61
+ "generation_args": {
62
+ "temperature": 0.6,
63
+ "top_p": 0.95,
64
+ "max_tokens": 65536,
65
+ "max_attempts": 1
66
+ },
67
+ "additional_details": {
68
+ "do_sample": "true",
69
+ "until": "[\"<|eot_id|>\", \"<|start_header_id|>user<|end_header_id|>\", \"Q:\", \"</s>\", \"<|im_end|>\"]",
70
+ "seed": "2",
71
+ "num_fewshot": "0"
72
+ }
73
+ }
74
+ },
75
+ {
76
+ "evaluation_name": "gsm8k_platinum_cot_llama/flexible-extract",
77
+ "source_data": {
78
+ "dataset_name": "gsm8k_platinum_cot_llama",
79
+ "source_type": "hf_dataset",
80
+ "hf_repo": "madrylab/gsm8k-platinum",
81
+ "hf_split": "test"
82
+ },
83
+ "metric_config": {
84
+ "evaluation_description": "exact_match (filter: flexible-extract)",
85
+ "lower_is_better": false,
86
+ "score_type": "continuous",
87
+ "min_score": 0.0,
88
+ "max_score": 1.0
89
+ },
90
+ "score_details": {
91
+ "score": 0.9509236283429832,
92
+ "details": {
93
+ "seed_scores": "[0.9520264681555004, 0.9495450785773366, 0.9511993382961125]",
94
+ "evaluation_timestamps": "[1782872444, 1782830712, 1782851542]",
95
+ "seed_values": "[2, 0, 1]"
96
+ },
97
+ "uncertainty": {
98
+ "standard_error": {
99
+ "value": 0.0007294599699654304,
100
+ "method": "across_seeds"
101
+ },
102
+ "num_samples": 3
103
+ }
104
+ },
105
+ "generation_config": {
106
+ "generation_args": {
107
+ "temperature": 0.6,
108
+ "top_p": 0.95,
109
+ "max_tokens": 65536,
110
+ "max_attempts": 1
111
+ },
112
+ "additional_details": {
113
+ "do_sample": "true",
114
+ "until": "[\"<|eot_id|>\", \"<|start_header_id|>user<|end_header_id|>\", \"Q:\", \"</s>\", \"<|im_end|>\"]",
115
+ "seed": "2",
116
+ "num_fewshot": "0"
117
+ }
118
+ }
119
+ }
120
+ ]
121
+ }
bf16_every_eval_ever/ifeval.json ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "ifeval/CohereLabs/command-a-plus-05-2026-bf16/1783690381.88",
4
+ "evaluation_timestamp": "1782846522",
5
+ "retrieved_timestamp": "1783690381.88",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "model_args": "{'model': 'CohereLabs/command-a-plus-05-2026-bf16', 'max_length': 72000, 'base_url': 'http://localhost:8000/v1/chat/completions', 'num_concurrent': 128, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 3600}",
26
+ "num_seeds_merged": "3"
27
+ }
28
+ },
29
+ "evaluation_results": [
30
+ {
31
+ "evaluation_name": "ifeval",
32
+ "source_data": {
33
+ "dataset_name": "ifeval",
34
+ "source_type": "hf_dataset",
35
+ "hf_repo": "google/IFEval",
36
+ "hf_split": "train"
37
+ },
38
+ "metric_config": {
39
+ "evaluation_description": "prompt_level_strict_acc",
40
+ "lower_is_better": false,
41
+ "score_type": "continuous",
42
+ "min_score": 0.0,
43
+ "max_score": 1.0
44
+ },
45
+ "score_details": {
46
+ "score": 0.8434996919285274,
47
+ "details": {
48
+ "seed_scores": "[0.8354898336414048, 0.8558225508317929, 0.8391866913123844]",
49
+ "evaluation_timestamps": "[1782867340, 1782888061, 1782846522]",
50
+ "seed_values": "[1, 2, 0]"
51
+ },
52
+ "uncertainty": {
53
+ "standard_error": {
54
+ "value": 0.0062531679390586525,
55
+ "method": "across_seeds"
56
+ },
57
+ "num_samples": 3
58
+ }
59
+ },
60
+ "generation_config": {
61
+ "generation_args": {
62
+ "temperature": 0.6,
63
+ "top_p": 0.95,
64
+ "max_tokens": 65536,
65
+ "max_attempts": 1
66
+ },
67
+ "additional_details": {
68
+ "until": "[]",
69
+ "do_sample": "true",
70
+ "seed": "1",
71
+ "num_fewshot": "0"
72
+ }
73
+ }
74
+ },
75
+ {
76
+ "evaluation_name": "ifeval",
77
+ "source_data": {
78
+ "dataset_name": "ifeval",
79
+ "source_type": "hf_dataset",
80
+ "hf_repo": "google/IFEval",
81
+ "hf_split": "train"
82
+ },
83
+ "metric_config": {
84
+ "evaluation_description": "inst_level_strict_acc",
85
+ "lower_is_better": false,
86
+ "score_type": "continuous",
87
+ "min_score": 0.0,
88
+ "max_score": 1.0
89
+ },
90
+ "score_details": {
91
+ "score": 0.864908073541167,
92
+ "details": {
93
+ "seed_scores": "[0.8561151079136691, 0.8776978417266187, 0.8609112709832134]",
94
+ "evaluation_timestamps": "[1782867340, 1782888061, 1782846522]",
95
+ "seed_values": "[1, 2, 0]"
96
+ },
97
+ "uncertainty": {
98
+ "standard_error": {
99
+ "value": 0.006543047779274537,
100
+ "method": "across_seeds"
101
+ },
102
+ "num_samples": 3
103
+ }
104
+ },
105
+ "generation_config": {
106
+ "generation_args": {
107
+ "temperature": 0.6,
108
+ "top_p": 0.95,
109
+ "max_tokens": 65536,
110
+ "max_attempts": 1
111
+ },
112
+ "additional_details": {
113
+ "until": "[]",
114
+ "do_sample": "true",
115
+ "seed": "1",
116
+ "num_fewshot": "0"
117
+ }
118
+ }
119
+ },
120
+ {
121
+ "evaluation_name": "ifeval",
122
+ "source_data": {
123
+ "dataset_name": "ifeval",
124
+ "source_type": "hf_dataset",
125
+ "hf_repo": "google/IFEval",
126
+ "hf_split": "train"
127
+ },
128
+ "metric_config": {
129
+ "evaluation_description": "prompt_level_loose_acc",
130
+ "lower_is_better": false,
131
+ "score_type": "continuous",
132
+ "min_score": 0.0,
133
+ "max_score": 1.0
134
+ },
135
+ "score_details": {
136
+ "score": 0.8589032655576093,
137
+ "details": {
138
+ "seed_scores": "[0.8539741219963032, 0.866913123844732, 0.8558225508317929]",
139
+ "evaluation_timestamps": "[1782867340, 1782888061, 1782846522]",
140
+ "seed_values": "[1, 2, 0]"
141
+ },
142
+ "uncertainty": {
143
+ "standard_error": {
144
+ "value": 0.0040403194850905655,
145
+ "method": "across_seeds"
146
+ },
147
+ "num_samples": 3
148
+ }
149
+ },
150
+ "generation_config": {
151
+ "generation_args": {
152
+ "temperature": 0.6,
153
+ "top_p": 0.95,
154
+ "max_tokens": 65536,
155
+ "max_attempts": 1
156
+ },
157
+ "additional_details": {
158
+ "until": "[]",
159
+ "do_sample": "true",
160
+ "seed": "1",
161
+ "num_fewshot": "0"
162
+ }
163
+ }
164
+ },
165
+ {
166
+ "evaluation_name": "ifeval",
167
+ "source_data": {
168
+ "dataset_name": "ifeval",
169
+ "source_type": "hf_dataset",
170
+ "hf_repo": "google/IFEval",
171
+ "hf_split": "train"
172
+ },
173
+ "metric_config": {
174
+ "evaluation_description": "inst_level_loose_acc",
175
+ "lower_is_better": false,
176
+ "score_type": "continuous",
177
+ "min_score": 0.0,
178
+ "max_score": 1.0
179
+ },
180
+ "score_details": {
181
+ "score": 0.8749000799360512,
182
+ "details": {
183
+ "seed_scores": "[0.86810551558753, 0.8848920863309353, 0.8717026378896883]",
184
+ "evaluation_timestamps": "[1782867340, 1782888061, 1782846522]",
185
+ "seed_values": "[1, 2, 0]"
186
+ },
187
+ "uncertainty": {
188
+ "standard_error": {
189
+ "value": 0.005102775913190935,
190
+ "method": "across_seeds"
191
+ },
192
+ "num_samples": 3
193
+ }
194
+ },
195
+ "generation_config": {
196
+ "generation_args": {
197
+ "temperature": 0.6,
198
+ "top_p": 0.95,
199
+ "max_tokens": 65536,
200
+ "max_attempts": 1
201
+ },
202
+ "additional_details": {
203
+ "until": "[]",
204
+ "do_sample": "true",
205
+ "seed": "1",
206
+ "num_fewshot": "0"
207
+ }
208
+ }
209
+ }
210
+ ]
211
+ }
bf16_every_eval_ever/math_500.json ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "math_500|0/CohereLabs/command-a-plus-05-2026-bf16/1783690408.011777",
4
+ "evaluation_timestamp": "777717",
5
+ "retrieved_timestamp": "1783690408.011777",
6
+ "source_metadata": {
7
+ "source_name": "lighteval",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lighteval",
14
+ "version": "0.13.1.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "provider": "hosted_vllm",
26
+ "base_url": "http://localhost:8000/v1",
27
+ "concurrent_requests": "16",
28
+ "verbose": "False",
29
+ "max_model_length": "131072",
30
+ "api_max_retry": "8",
31
+ "api_retry_sleep": "1.0",
32
+ "api_retry_multiplier": "2.0",
33
+ "timeout": "3600.0",
34
+ "num_seeds_merged": "3"
35
+ }
36
+ },
37
+ "evaluation_results": [
38
+ {
39
+ "evaluation_name": "math_500",
40
+ "source_data": {
41
+ "dataset_name": "math_500",
42
+ "source_type": "hf_dataset",
43
+ "hf_repo": "HuggingFaceH4/MATH-500",
44
+ "hf_split": "test"
45
+ },
46
+ "metric_config": {
47
+ "evaluation_description": "pass@k:k=1&n=1",
48
+ "lower_is_better": false,
49
+ "score_type": "continuous",
50
+ "min_score": 0.0,
51
+ "max_score": 1.0
52
+ },
53
+ "score_details": {
54
+ "score": 0.898,
55
+ "details": {
56
+ "seed_scores": "[0.902, 0.882, 0.91]",
57
+ "evaluation_timestamps": "[782779, 787744, 777717]",
58
+ "seed_values": "[1, 2, 0]"
59
+ },
60
+ "uncertainty": {
61
+ "standard_error": {
62
+ "value": 0.008326663997864539,
63
+ "method": "across_seeds"
64
+ },
65
+ "num_samples": 3
66
+ }
67
+ },
68
+ "generation_config": {
69
+ "generation_args": {
70
+ "temperature": 0.6,
71
+ "top_p": 0.95,
72
+ "max_tokens": 65536,
73
+ "max_attempts": 1
74
+ },
75
+ "additional_details": {
76
+ "seed": "1",
77
+ "num_fewshot": "0"
78
+ }
79
+ }
80
+ }
81
+ ]
82
+ }
bf16_every_eval_ever/mmlu_cot_llama.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "mmlu_cot_llama/CohereLabs/command-a-plus-05-2026-bf16/1783690381.884638",
4
+ "evaluation_timestamp": "1782847962",
5
+ "retrieved_timestamp": "1783690381.884638",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "model_args": "{'model': 'CohereLabs/command-a-plus-05-2026-bf16', 'max_length': 72000, 'base_url': 'http://localhost:8000/v1/chat/completions', 'num_concurrent': 128, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 3600}",
26
+ "num_seeds_merged": "3"
27
+ }
28
+ },
29
+ "evaluation_results": [
30
+ {
31
+ "evaluation_name": "mmlu_cot_llama/strict_match",
32
+ "source_data": {
33
+ "dataset_name": "mmlu_cot_llama",
34
+ "source_type": "other"
35
+ },
36
+ "metric_config": {
37
+ "evaluation_description": "exact_match (filter: strict_match)",
38
+ "lower_is_better": false,
39
+ "score_type": "continuous",
40
+ "min_score": 0.0,
41
+ "max_score": 1.0
42
+ },
43
+ "score_details": {
44
+ "score": 0.827517447657029,
45
+ "details": {
46
+ "seed_scores": "[0.8277310924369747, 0.827303802877083, 0.827517447657029]",
47
+ "evaluation_timestamps": "[1782868757, 1782889375, 1782847962]"
48
+ },
49
+ "uncertainty": {
50
+ "standard_error": {
51
+ "value": 0.0001233478712126908,
52
+ "method": "across_seeds"
53
+ },
54
+ "num_samples": 3
55
+ }
56
+ }
57
+ }
58
+ ]
59
+ }
bf16_every_eval_ever/mmlu_pro_chat.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": "0.2.2",
3
+ "evaluation_id": "mmlu_pro_chat/CohereLabs/command-a-plus-05-2026-bf16/1783690381.888367",
4
+ "evaluation_timestamp": "1782831148",
5
+ "retrieved_timestamp": "1783690381.888367",
6
+ "source_metadata": {
7
+ "source_name": "lm-evaluation-harness",
8
+ "source_type": "evaluation_run",
9
+ "source_organization_name": "RedHatAI",
10
+ "evaluator_relationship": "third_party"
11
+ },
12
+ "eval_library": {
13
+ "name": "lm_eval",
14
+ "version": "0.4.13.dev0"
15
+ },
16
+ "model_info": {
17
+ "name": "CohereLabs/command-a-plus-05-2026-bf16",
18
+ "id": "CohereLabs/command-a-plus-05-2026-bf16",
19
+ "developer": "CohereLabs",
20
+ "inference_engine": {
21
+ "name": "vllm",
22
+ "version": "0.22.1"
23
+ },
24
+ "additional_details": {
25
+ "model_args": "{'model': 'CohereLabs/command-a-plus-05-2026-bf16', 'max_length': 72000, 'base_url': 'http://localhost:8000/v1/chat/completions', 'num_concurrent': 128, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 3600}",
26
+ "num_seeds_merged": "3"
27
+ }
28
+ },
29
+ "evaluation_results": [
30
+ {
31
+ "evaluation_name": "mmlu_pro_chat/custom-extract",
32
+ "source_data": {
33
+ "dataset_name": "mmlu_pro_chat",
34
+ "source_type": "other"
35
+ },
36
+ "metric_config": {
37
+ "evaluation_description": "exact_match (filter: custom-extract)",
38
+ "lower_is_better": false,
39
+ "score_type": "continuous",
40
+ "min_score": 0.0,
41
+ "max_score": 1.0
42
+ },
43
+ "score_details": {
44
+ "score": 0.7775653812056738,
45
+ "details": {
46
+ "seed_scores": "[0.7788397606382979, 0.7774268617021277, 0.7764295212765957]",
47
+ "evaluation_timestamps": "[1782831148, 1782851687, 1782872583]"
48
+ },
49
+ "uncertainty": {
50
+ "standard_error": {
51
+ "value": 0.000699214841762191,
52
+ "method": "across_seeds"
53
+ },
54
+ "num_samples": 3
55
+ }
56
+ }
57
+ }
58
+ ]
59
+ }