Maggio33 commited on
Commit
bb637c3
Β·
verified Β·
1 Parent(s): 0170c51

Slayer1B 1.08: update cards and documentation

Browse files
Files changed (2) hide show
  1. EVALUATION.json +551 -0
  2. README.md +30 -33
EVALUATION.json ADDED
@@ -0,0 +1,551 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "measurement_owner": "SlayerLab / Fabryka AI",
4
+ "status": "descriptive; training-lineage contamination scan pending",
5
+ "source_aggregate_sha256": "ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db",
6
+ "source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
7
+ "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
8
+ "community_report_is_separate_measurement": true,
9
+ "protocol": {
10
+ "few_shot": 0,
11
+ "dtype": "float32",
12
+ "context_limit": 4096,
13
+ "initial_token_id": 0,
14
+ "special_tokens": false,
15
+ "truncation": "left",
16
+ "scoring": "summed continuation-token log likelihood; task-specific partial scoring for WinoGrande",
17
+ "metric_units": "fraction; multiply by 100 for card percentages",
18
+ "primary_metrics": {
19
+ "arc_challenge": "acc_norm",
20
+ "hellaswag": "acc_norm",
21
+ "piqa": "acc_norm",
22
+ "winogrande": "acc",
23
+ "boolq": "acc",
24
+ "openbookqa": "acc_norm",
25
+ "belebele_pol": "acc",
26
+ "hellaswag_pl": "acc_norm",
27
+ "mmlu_pl": "acc"
28
+ },
29
+ "mmlu_aggregation": "micro-average over questions",
30
+ "polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b"
31
+ },
32
+ "ci": {
33
+ "level": 0.95,
34
+ "method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
35
+ "replicates": 10000,
36
+ "base_seed": 20261110,
37
+ "seed_scope": "base seed plus model label, as source generator",
38
+ "assumption": "items treated as independent; no subject, template or source clusters",
39
+ "interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
40
+ },
41
+ "models": {
42
+ "1.07": {
43
+ "step": 22327,
44
+ "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
45
+ "weights_published": true,
46
+ "source_results_sha256": "9d89a9cdc1ea66723c9c709930d987900609bee84de5598dba400231407babf2",
47
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
48
+ "tasks": {
49
+ "arc_challenge": {
50
+ "n": 1172,
51
+ "n_pelne": 1172,
52
+ "dataset_repo": "allenai/ai2_arc",
53
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
54
+ "pliki_sha256": {
55
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
56
+ },
57
+ "acc": 0.261945,
58
+ "acc_ci95": [
59
+ 0.237201,
60
+ 0.286689
61
+ ],
62
+ "acc_norm": 0.273038,
63
+ "acc_norm_ci95": [
64
+ 0.24744,
65
+ 0.298635
66
+ ],
67
+ "split": "test",
68
+ "correct_primary": 320
69
+ },
70
+ "hellaswag": {
71
+ "n": 10042,
72
+ "n_pelne": 10042,
73
+ "dataset_repo": "Rowan/hellaswag",
74
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
75
+ "pliki_sha256": {
76
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
77
+ },
78
+ "acc": 0.351324,
79
+ "acc_ci95": [
80
+ 0.341762,
81
+ 0.360486
82
+ ],
83
+ "acc_norm": 0.433778,
84
+ "acc_norm_ci95": [
85
+ 0.424318,
86
+ 0.443438
87
+ ],
88
+ "split": "validation",
89
+ "correct_primary": 4356
90
+ },
91
+ "piqa": {
92
+ "n": 1838,
93
+ "n_pelne": 1838,
94
+ "dataset_repo": "baber/piqa",
95
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
96
+ "pliki_sha256": {
97
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
98
+ },
99
+ "acc": 0.670294,
100
+ "acc_ci95": [
101
+ 0.648531,
102
+ 0.691526
103
+ ],
104
+ "acc_norm": 0.66975,
105
+ "acc_norm_ci95": [
106
+ 0.647987,
107
+ 0.691513
108
+ ],
109
+ "split": "validation",
110
+ "correct_primary": 1231
111
+ },
112
+ "winogrande": {
113
+ "n": 1267,
114
+ "n_pelne": 1267,
115
+ "dataset_repo": "allenai/winogrande",
116
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
117
+ "pliki_sha256": {
118
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
119
+ },
120
+ "acc": 0.535912,
121
+ "acc_ci95": [
122
+ 0.508287,
123
+ 0.563536
124
+ ],
125
+ "acc_norm": 0.535912,
126
+ "acc_norm_ci95": [
127
+ 0.509077,
128
+ 0.563536
129
+ ],
130
+ "split": "validation",
131
+ "correct_primary": 679
132
+ },
133
+ "boolq": {
134
+ "n": 3270,
135
+ "n_pelne": 3270,
136
+ "dataset_repo": "aps/super_glue",
137
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
138
+ "pliki_sha256": {
139
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
140
+ },
141
+ "acc": 0.582875,
142
+ "acc_ci95": [
143
+ 0.566055,
144
+ 0.599694
145
+ ],
146
+ "acc_norm": 0.621101,
147
+ "acc_norm_ci95": [
148
+ 0.604281,
149
+ 0.637615
150
+ ],
151
+ "split": "validation",
152
+ "correct_primary": 1906
153
+ },
154
+ "openbookqa": {
155
+ "n": 500,
156
+ "n_pelne": 500,
157
+ "dataset_repo": "allenai/openbookqa",
158
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
159
+ "pliki_sha256": {
160
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
161
+ },
162
+ "acc": 0.2,
163
+ "acc_ci95": [
164
+ 0.166,
165
+ 0.236
166
+ ],
167
+ "acc_norm": 0.336,
168
+ "acc_norm_ci95": [
169
+ 0.294,
170
+ 0.378
171
+ ],
172
+ "split": "test",
173
+ "correct_primary": 168
174
+ },
175
+ "belebele_pol": {
176
+ "n": 900,
177
+ "n_pelne": 900,
178
+ "dataset_repo": "facebook/belebele",
179
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
180
+ "pliki_sha256": {
181
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
182
+ },
183
+ "acc": 0.26,
184
+ "acc_ci95": [
185
+ 0.231111,
186
+ 0.288889
187
+ ],
188
+ "acc_norm": 0.26,
189
+ "acc_norm_ci95": [
190
+ 0.232222,
191
+ 0.288889
192
+ ],
193
+ "split": "pol_Latn evaluation set",
194
+ "correct_primary": 234
195
+ },
196
+ "hellaswag_pl": {
197
+ "n": 10042,
198
+ "n_pelne": 10042,
199
+ "dataset_repo": "Eurolingua/hellaswagx",
200
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
201
+ "pliki_sha256": {
202
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
203
+ },
204
+ "acc": 0.345051,
205
+ "acc_ci95": [
206
+ 0.335986,
207
+ 0.354611
208
+ ],
209
+ "acc_norm": 0.438259,
210
+ "acc_norm_ci95": [
211
+ 0.428401,
212
+ 0.447921
213
+ ],
214
+ "split": "validation",
215
+ "correct_primary": 4401
216
+ },
217
+ "mmlu_pl": {
218
+ "n": 14042,
219
+ "n_pelne": 14042,
220
+ "dataset_repo": "Eurolingua/mmlux",
221
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
222
+ "pliki_sha256": {
223
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
224
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
225
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
226
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
227
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
228
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
229
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
230
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
231
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
232
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
233
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
234
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
235
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
236
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
237
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
238
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
239
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
240
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
241
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
242
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
243
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
244
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
245
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
246
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
247
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
248
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
249
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
250
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
251
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
252
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
253
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
254
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
255
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
256
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
257
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
258
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
259
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
260
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
261
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
262
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
263
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
264
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
265
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
266
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
267
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
268
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
269
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
270
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
271
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
272
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
273
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
274
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
275
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
276
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
277
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
278
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
279
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
280
+ },
281
+ "acc": 0.240137,
282
+ "acc_ci95": [
283
+ 0.233015,
284
+ 0.247116
285
+ ],
286
+ "acc_norm": 0.240137,
287
+ "acc_norm_ci95": [
288
+ 0.233158,
289
+ 0.247187
290
+ ],
291
+ "split": "test; micro-average over subjects",
292
+ "correct_primary": 3372
293
+ }
294
+ }
295
+ },
296
+ "1.08": {
297
+ "step": 1709,
298
+ "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
299
+ "weights_published": true,
300
+ "source_results_sha256": "108e43d0ac726f946c0480ff00319eb1811a3e254f05b1540156b0db11d1566e",
301
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
302
+ "tasks": {
303
+ "arc_challenge": {
304
+ "n": 1172,
305
+ "n_pelne": 1172,
306
+ "dataset_repo": "allenai/ai2_arc",
307
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
308
+ "pliki_sha256": {
309
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
310
+ },
311
+ "acc": 0.271331,
312
+ "acc_ci95": [
313
+ 0.245734,
314
+ 0.296928
315
+ ],
316
+ "acc_norm": 0.300341,
317
+ "acc_norm_ci95": [
318
+ 0.273891,
319
+ 0.326792
320
+ ],
321
+ "split": "test",
322
+ "correct_primary": 352
323
+ },
324
+ "hellaswag": {
325
+ "n": 10042,
326
+ "n_pelne": 10042,
327
+ "dataset_repo": "Rowan/hellaswag",
328
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
329
+ "pliki_sha256": {
330
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
331
+ },
332
+ "acc": 0.350926,
333
+ "acc_ci95": [
334
+ 0.341765,
335
+ 0.360486
336
+ ],
337
+ "acc_norm": 0.434176,
338
+ "acc_norm_ci95": [
339
+ 0.424318,
340
+ 0.443935
341
+ ],
342
+ "split": "validation",
343
+ "correct_primary": 4360
344
+ },
345
+ "piqa": {
346
+ "n": 1838,
347
+ "n_pelne": 1838,
348
+ "dataset_repo": "baber/piqa",
349
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
350
+ "pliki_sha256": {
351
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
352
+ },
353
+ "acc": 0.681719,
354
+ "acc_ci95": [
355
+ 0.661045,
356
+ 0.702938
357
+ ],
358
+ "acc_norm": 0.662133,
359
+ "acc_norm_ci95": [
360
+ 0.641458,
361
+ 0.683351
362
+ ],
363
+ "split": "validation",
364
+ "correct_primary": 1217
365
+ },
366
+ "winogrande": {
367
+ "n": 1267,
368
+ "n_pelne": 1267,
369
+ "dataset_repo": "allenai/winogrande",
370
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
371
+ "pliki_sha256": {
372
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
373
+ },
374
+ "acc": 0.536701,
375
+ "acc_ci95": [
376
+ 0.509077,
377
+ 0.564325
378
+ ],
379
+ "acc_norm": 0.536701,
380
+ "acc_norm_ci95": [
381
+ 0.508287,
382
+ 0.563536
383
+ ],
384
+ "split": "validation",
385
+ "correct_primary": 680
386
+ },
387
+ "boolq": {
388
+ "n": 3270,
389
+ "n_pelne": 3270,
390
+ "dataset_repo": "aps/super_glue",
391
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
392
+ "pliki_sha256": {
393
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
394
+ },
395
+ "acc": 0.599694,
396
+ "acc_ci95": [
397
+ 0.582569,
398
+ 0.615902
399
+ ],
400
+ "acc_norm": 0.620489,
401
+ "acc_norm_ci95": [
402
+ 0.603364,
403
+ 0.637309
404
+ ],
405
+ "split": "validation",
406
+ "correct_primary": 1961
407
+ },
408
+ "openbookqa": {
409
+ "n": 500,
410
+ "n_pelne": 500,
411
+ "dataset_repo": "allenai/openbookqa",
412
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
413
+ "pliki_sha256": {
414
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
415
+ },
416
+ "acc": 0.23,
417
+ "acc_ci95": [
418
+ 0.194,
419
+ 0.266
420
+ ],
421
+ "acc_norm": 0.348,
422
+ "acc_norm_ci95": [
423
+ 0.306,
424
+ 0.39
425
+ ],
426
+ "split": "test",
427
+ "correct_primary": 174
428
+ },
429
+ "belebele_pol": {
430
+ "n": 900,
431
+ "n_pelne": 900,
432
+ "dataset_repo": "facebook/belebele",
433
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
434
+ "pliki_sha256": {
435
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
436
+ },
437
+ "acc": 0.24,
438
+ "acc_ci95": [
439
+ 0.212222,
440
+ 0.268889
441
+ ],
442
+ "acc_norm": 0.24,
443
+ "acc_norm_ci95": [
444
+ 0.212222,
445
+ 0.267778
446
+ ],
447
+ "split": "pol_Latn evaluation set",
448
+ "correct_primary": 216
449
+ },
450
+ "hellaswag_pl": {
451
+ "n": 10042,
452
+ "n_pelne": 10042,
453
+ "dataset_repo": "Eurolingua/hellaswagx",
454
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
455
+ "pliki_sha256": {
456
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
457
+ },
458
+ "acc": 0.352221,
459
+ "acc_ci95": [
460
+ 0.342661,
461
+ 0.361581
462
+ ],
463
+ "acc_norm": 0.438956,
464
+ "acc_norm_ci95": [
465
+ 0.428998,
466
+ 0.448715
467
+ ],
468
+ "split": "validation",
469
+ "correct_primary": 4408
470
+ },
471
+ "mmlu_pl": {
472
+ "n": 14042,
473
+ "n_pelne": 14042,
474
+ "dataset_repo": "Eurolingua/mmlux",
475
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
476
+ "pliki_sha256": {
477
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
478
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
479
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
480
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
481
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
482
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
483
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
484
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
485
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
486
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
487
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
488
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
489
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
490
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
491
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
492
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
493
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
494
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
495
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
496
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
497
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
498
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
499
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
500
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
501
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
502
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
503
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
504
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
505
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
506
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
507
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
508
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
509
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
510
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
511
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
512
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
513
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
514
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
515
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
516
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
517
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
518
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
519
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
520
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
521
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
522
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
523
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
524
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
525
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
526
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
527
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
528
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
529
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
530
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
531
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
532
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
533
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
534
+ },
535
+ "acc": 0.23038,
536
+ "acc_ci95": [
537
+ 0.223401,
538
+ 0.237431
539
+ ],
540
+ "acc_norm": 0.23038,
541
+ "acc_norm_ci95": [
542
+ 0.223259,
543
+ 0.237288
544
+ ],
545
+ "split": "test; micro-average over subjects",
546
+ "correct_primary": 3235
547
+ }
548
+ }
549
+ }
550
+ }
551
+ }
README.md CHANGED
@@ -10,14 +10,13 @@ tags:
10
  - causal-lm
11
  - continued-pretraining
12
  - experimental
13
- - guard-failed
14
  ---
15
 
16
  # Slayer1B 1.08
17
 
18
  **Arkadiusz SΕ‚ota, Anna Sieczka, Piotr StyΕ‚a / Fabryka AI (SlayerLab)**
19
 
20
- **Experimental checkpoint β€” acceptance guard FAIL, 9 November 2026.** Three preregistered endpoint gains are confirmed, but the ARC-Easy retention guard fails. This is **not an accepted successor to 1.07**. The new 1.08 is distinct from the earlier 1.08b experiment.
21
 
22
  [Public training trace](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a).
23
 
@@ -25,35 +24,6 @@ tags:
25
 
26
  Train on as many approved token positions as the fixed compute budget permits, recording actual progress and expense at the spot prices observed during the run. Synthetic speed, real-data training and downstream evaluation are separate measurements. The expense report covers the billed session, including preparation and evaluation; steady-state cost per billion tokens is an extrapolation, not that bill.
27
 
28
- ## Pre-registered evaluation and outcome
29
-
30
- FINAL and 1.07 were evaluated on the same B200 hardware. Reported 95% CIs use paired bootstrap with 10,000 draws; one-sided p-values use Holm correction over three confirmatory endpoints. One run means sampling uncertainty, not training-seed variance. Mathematics uses base mode, three-shot, strict pass@1; its baseline exceeds the prespecified 5% validity threshold. Domain bpb uses matched fixed samples of at most two million native tokens, scored per UTF-8 byte.
31
-
32
- | Confirmatory endpoint | n | 1.07 | 1.08 | Difference [95% CI] | Holm p |
33
- |---|---:|---:|---:|---|---:|
34
- | Polish mathematics pass@1 | 300 items | 6.33% | 12.00% | +5.67 pp [+1.67, +9.67] | 0.0035 |
35
- | Polish logic bpb | 314 documents | 0.78727 | 0.70895 | βˆ’0.07832 [βˆ’0.09000, βˆ’0.06865] | 0.0003 |
36
- | Within-source English technical VALID bpb | 487 documents | 0.66661 | 0.38549 | βˆ’0.28112 [βˆ’0.34537, βˆ’0.23652] | 0.0003 |
37
-
38
- All three endpoints pass their confirmatory tests. Technical VALID shares source repositories with TRAIN; only exact text duplicates were removed. Its gain is within training-source projects, **not evidence of generalization to unseen technical repositories**.
39
-
40
- | Preregistered guard | n | 1.07 | 1.08 | Difference | Allowed difference | Outcome |
41
- |---|---:|---:|---:|---:|---|---|
42
- | ARC-Easy | 2,376 | 58.67% | 56.19% | βˆ’2.48 pp | β‰₯ βˆ’1.00 pp | **FAIL** |
43
- | BLiMP | 67,000 | 77.90% | 76.91% | βˆ’0.99 pp | β‰₯ βˆ’1.00 pp | PASS, near boundary |
44
- | ARC-Easy-PL | 2,376 | 46.93% | 47.14% | +0.21 pp | β‰₯ βˆ’1.00 pp | PASS |
45
- | MultiBLiMP-PL | 3,272 | 98.84% | 98.47% | βˆ’0.37 pp | β‰₯ βˆ’1.00 pp | PASS |
46
- | Polish H bpb | 1,056 documents | 0.75459 | 0.74835 | βˆ’0.00624 | ≀ +0.005 | PASS |
47
-
48
- The frozen rule makes **any guard failure prevent acceptance**; endpoint gains do not override it.
49
-
50
- | Descriptive diagnostic, not confirmatory | n | 1.07 | 1.08 |
51
- |---|---:|---:|---:|
52
- | Held-out reasoning v2 pass@1 | 360 | 13.33% | 9.44% |
53
- | Arithmetic grid pass@1 | 500 | 12.20% | 21.20% |
54
- | WikiText-2 byte perplexity | 1,292,008 bytes | 2.1837 | 2.1939 |
55
- | Polish H hit-free subset bpb | 711 documents | 0.78425 | 0.77805 |
56
-
57
  ## Model and starting point
58
 
59
  The run initializes from [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md), revision `a090b54e658fb57dc5166c1fcc90cc6bf5c80106`. Its published card describes a Polish-dominant decoder-only model with **1,073,912,851 parameters**, a **32,768-token v6 vocabulary**, and final checkpoint **22,327**. These are base-model properties.
@@ -82,9 +52,10 @@ The continuation preserves that architecture and tokenizer. Its task is causal t
82
  | Steady interval throughput | Approximately 572,000 positions/s; not positions divided by the whole training window |
83
  | Training loss | Approximately 2.20 at update 10 to 1.99 at completion; not an evaluation score |
84
  | Training GPU cost | Approximately USD 7.73, or USD 4.32 per billion processed positions, calculated from the observed window and spot rate |
85
- | Attributed session cost | Approximately USD 21.5 through evaluation completion; preparation β‰ˆ11.2, training 7.73, evaluation/export β‰ˆ2.6. Provider-counter attribution, not a separate settled invoice; the VM continues under 1.09 |
 
86
 
87
- Input capacity, processed positions, steady interval speed and the complete billed session remain separate quantities. The failed acceptance verdict is unchanged by permission to publish this experiment.
88
 
89
  ## Training inputs
90
 
@@ -102,6 +73,32 @@ A replay-only continued-pretraining control is not budgeted for the priority run
102
 
103
  Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
104
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  ## Licensing and limitations
106
 
107
  Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.
 
10
  - causal-lm
11
  - continued-pretraining
12
  - experimental
 
13
  ---
14
 
15
  # Slayer1B 1.08
16
 
17
  **Arkadiusz SΕ‚ota, Anna Sieczka, Piotr StyΕ‚a / Fabryka AI (SlayerLab)**
18
 
19
+ **Slayer1B 1.08 is an experimental base language model continued from Slayer1B 1.07.** Training completed on 9 November 2026 after 1,709 updates and 1,792,016,384 processed token positions, using Polish analytical text, English technical text and replay. It is not instruction-tuned and is distinct from the earlier 1.08b ablation.
20
 
21
  [Public training trace](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a).
22
 
 
24
 
25
  Train on as many approved token positions as the fixed compute budget permits, recording actual progress and expense at the spot prices observed during the run. Synthetic speed, real-data training and downstream evaluation are separate measurements. The expense report covers the billed session, including preparation and evaluation; steady-state cost per billion tokens is an extrapolation, not that bill.
26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  ## Model and starting point
28
 
29
  The run initializes from [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md), revision `a090b54e658fb57dc5166c1fcc90cc6bf5c80106`. Its published card describes a Polish-dominant decoder-only model with **1,073,912,851 parameters**, a **32,768-token v6 vocabulary**, and final checkpoint **22,327**. These are base-model properties.
 
52
  | Steady interval throughput | Approximately 572,000 positions/s; not positions divided by the whole training window |
53
  | Training loss | Approximately 2.20 at update 10 to 1.99 at completion; not an evaluation score |
54
  | Training GPU cost | Approximately USD 7.73, or USD 4.32 per billion processed positions, calculated from the observed window and spot rate |
55
+ | Cost through evaluation completion | Approximately USD 21.5; preparation β‰ˆ11.2, training 7.73, evaluation/export β‰ˆ2.6. Provider-counter attribution, not a separate settled invoice |
56
+ | Complete allocation cost | Approximately USD 24.1; the allocation was preempted shortly afterwards and released, rather than reused for 1.09 |
57
 
58
+ Input capacity, processed positions, steady interval speed and the complete billed session remain separate quantities.
59
 
60
  ## Training inputs
61
 
 
73
 
74
  Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
75
 
76
+ ## Standard benchmark measurements
77
+
78
+ We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol work and evaluation proposals. The team's independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is a separate measurement source. **The table below reports SlayerLab evaluations**, also published in the [1.07 training record](https://track.fabryka.ai/run/4973c068-8dd5-59d8-b6c4-671ca3beb925) and the [1.08 training record](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a); it is not a copy of that independent run.
79
+
80
+ All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
81
+
82
+ | Task | Metric | n | 1.07, update 22,327 | 1.08, update 1,709 |
83
+ |---|---|---:|---:|---:|
84
+ | ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] |
85
+ | HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] |
86
+ | PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] |
87
+ | WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] |
88
+ | BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] |
89
+ | OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] |
90
+ | Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] |
91
+ | HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] |
92
+ | MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] |
93
+
94
+ Intervals use the percentile bootstrap with 10,000 replicates, seed base 20261110 and a model-specific stream. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
95
+
96
+ [Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
97
+
98
+ **Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
99
+
100
+ Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
101
+
102
  ## Licensing and limitations
103
 
104
  Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.