Slayer1B 1.08: update cards and documentation
Browse files- EVALUATION.json +551 -0
- README.md +30 -33
EVALUATION.json
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
+
"status": "descriptive; training-lineage contamination scan pending",
|
| 5 |
+
"source_aggregate_sha256": "ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db",
|
| 6 |
+
"source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
|
| 7 |
+
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
+
"community_report_is_separate_measurement": true,
|
| 9 |
+
"protocol": {
|
| 10 |
+
"few_shot": 0,
|
| 11 |
+
"dtype": "float32",
|
| 12 |
+
"context_limit": 4096,
|
| 13 |
+
"initial_token_id": 0,
|
| 14 |
+
"special_tokens": false,
|
| 15 |
+
"truncation": "left",
|
| 16 |
+
"scoring": "summed continuation-token log likelihood; task-specific partial scoring for WinoGrande",
|
| 17 |
+
"metric_units": "fraction; multiply by 100 for card percentages",
|
| 18 |
+
"primary_metrics": {
|
| 19 |
+
"arc_challenge": "acc_norm",
|
| 20 |
+
"hellaswag": "acc_norm",
|
| 21 |
+
"piqa": "acc_norm",
|
| 22 |
+
"winogrande": "acc",
|
| 23 |
+
"boolq": "acc",
|
| 24 |
+
"openbookqa": "acc_norm",
|
| 25 |
+
"belebele_pol": "acc",
|
| 26 |
+
"hellaswag_pl": "acc_norm",
|
| 27 |
+
"mmlu_pl": "acc"
|
| 28 |
+
},
|
| 29 |
+
"mmlu_aggregation": "micro-average over questions",
|
| 30 |
+
"polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b"
|
| 31 |
+
},
|
| 32 |
+
"ci": {
|
| 33 |
+
"level": 0.95,
|
| 34 |
+
"method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
|
| 35 |
+
"replicates": 10000,
|
| 36 |
+
"base_seed": 20261110,
|
| 37 |
+
"seed_scope": "base seed plus model label, as source generator",
|
| 38 |
+
"assumption": "items treated as independent; no subject, template or source clusters",
|
| 39 |
+
"interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
|
| 40 |
+
},
|
| 41 |
+
"models": {
|
| 42 |
+
"1.07": {
|
| 43 |
+
"step": 22327,
|
| 44 |
+
"weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
|
| 45 |
+
"weights_published": true,
|
| 46 |
+
"source_results_sha256": "9d89a9cdc1ea66723c9c709930d987900609bee84de5598dba400231407babf2",
|
| 47 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 48 |
+
"tasks": {
|
| 49 |
+
"arc_challenge": {
|
| 50 |
+
"n": 1172,
|
| 51 |
+
"n_pelne": 1172,
|
| 52 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 53 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 54 |
+
"pliki_sha256": {
|
| 55 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 56 |
+
},
|
| 57 |
+
"acc": 0.261945,
|
| 58 |
+
"acc_ci95": [
|
| 59 |
+
0.237201,
|
| 60 |
+
0.286689
|
| 61 |
+
],
|
| 62 |
+
"acc_norm": 0.273038,
|
| 63 |
+
"acc_norm_ci95": [
|
| 64 |
+
0.24744,
|
| 65 |
+
0.298635
|
| 66 |
+
],
|
| 67 |
+
"split": "test",
|
| 68 |
+
"correct_primary": 320
|
| 69 |
+
},
|
| 70 |
+
"hellaswag": {
|
| 71 |
+
"n": 10042,
|
| 72 |
+
"n_pelne": 10042,
|
| 73 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 74 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 75 |
+
"pliki_sha256": {
|
| 76 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 77 |
+
},
|
| 78 |
+
"acc": 0.351324,
|
| 79 |
+
"acc_ci95": [
|
| 80 |
+
0.341762,
|
| 81 |
+
0.360486
|
| 82 |
+
],
|
| 83 |
+
"acc_norm": 0.433778,
|
| 84 |
+
"acc_norm_ci95": [
|
| 85 |
+
0.424318,
|
| 86 |
+
0.443438
|
| 87 |
+
],
|
| 88 |
+
"split": "validation",
|
| 89 |
+
"correct_primary": 4356
|
| 90 |
+
},
|
| 91 |
+
"piqa": {
|
| 92 |
+
"n": 1838,
|
| 93 |
+
"n_pelne": 1838,
|
| 94 |
+
"dataset_repo": "baber/piqa",
|
| 95 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 96 |
+
"pliki_sha256": {
|
| 97 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 98 |
+
},
|
| 99 |
+
"acc": 0.670294,
|
| 100 |
+
"acc_ci95": [
|
| 101 |
+
0.648531,
|
| 102 |
+
0.691526
|
| 103 |
+
],
|
| 104 |
+
"acc_norm": 0.66975,
|
| 105 |
+
"acc_norm_ci95": [
|
| 106 |
+
0.647987,
|
| 107 |
+
0.691513
|
| 108 |
+
],
|
| 109 |
+
"split": "validation",
|
| 110 |
+
"correct_primary": 1231
|
| 111 |
+
},
|
| 112 |
+
"winogrande": {
|
| 113 |
+
"n": 1267,
|
| 114 |
+
"n_pelne": 1267,
|
| 115 |
+
"dataset_repo": "allenai/winogrande",
|
| 116 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 117 |
+
"pliki_sha256": {
|
| 118 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 119 |
+
},
|
| 120 |
+
"acc": 0.535912,
|
| 121 |
+
"acc_ci95": [
|
| 122 |
+
0.508287,
|
| 123 |
+
0.563536
|
| 124 |
+
],
|
| 125 |
+
"acc_norm": 0.535912,
|
| 126 |
+
"acc_norm_ci95": [
|
| 127 |
+
0.509077,
|
| 128 |
+
0.563536
|
| 129 |
+
],
|
| 130 |
+
"split": "validation",
|
| 131 |
+
"correct_primary": 679
|
| 132 |
+
},
|
| 133 |
+
"boolq": {
|
| 134 |
+
"n": 3270,
|
| 135 |
+
"n_pelne": 3270,
|
| 136 |
+
"dataset_repo": "aps/super_glue",
|
| 137 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 138 |
+
"pliki_sha256": {
|
| 139 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 140 |
+
},
|
| 141 |
+
"acc": 0.582875,
|
| 142 |
+
"acc_ci95": [
|
| 143 |
+
0.566055,
|
| 144 |
+
0.599694
|
| 145 |
+
],
|
| 146 |
+
"acc_norm": 0.621101,
|
| 147 |
+
"acc_norm_ci95": [
|
| 148 |
+
0.604281,
|
| 149 |
+
0.637615
|
| 150 |
+
],
|
| 151 |
+
"split": "validation",
|
| 152 |
+
"correct_primary": 1906
|
| 153 |
+
},
|
| 154 |
+
"openbookqa": {
|
| 155 |
+
"n": 500,
|
| 156 |
+
"n_pelne": 500,
|
| 157 |
+
"dataset_repo": "allenai/openbookqa",
|
| 158 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 159 |
+
"pliki_sha256": {
|
| 160 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 161 |
+
},
|
| 162 |
+
"acc": 0.2,
|
| 163 |
+
"acc_ci95": [
|
| 164 |
+
0.166,
|
| 165 |
+
0.236
|
| 166 |
+
],
|
| 167 |
+
"acc_norm": 0.336,
|
| 168 |
+
"acc_norm_ci95": [
|
| 169 |
+
0.294,
|
| 170 |
+
0.378
|
| 171 |
+
],
|
| 172 |
+
"split": "test",
|
| 173 |
+
"correct_primary": 168
|
| 174 |
+
},
|
| 175 |
+
"belebele_pol": {
|
| 176 |
+
"n": 900,
|
| 177 |
+
"n_pelne": 900,
|
| 178 |
+
"dataset_repo": "facebook/belebele",
|
| 179 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 180 |
+
"pliki_sha256": {
|
| 181 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 182 |
+
},
|
| 183 |
+
"acc": 0.26,
|
| 184 |
+
"acc_ci95": [
|
| 185 |
+
0.231111,
|
| 186 |
+
0.288889
|
| 187 |
+
],
|
| 188 |
+
"acc_norm": 0.26,
|
| 189 |
+
"acc_norm_ci95": [
|
| 190 |
+
0.232222,
|
| 191 |
+
0.288889
|
| 192 |
+
],
|
| 193 |
+
"split": "pol_Latn evaluation set",
|
| 194 |
+
"correct_primary": 234
|
| 195 |
+
},
|
| 196 |
+
"hellaswag_pl": {
|
| 197 |
+
"n": 10042,
|
| 198 |
+
"n_pelne": 10042,
|
| 199 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 200 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 201 |
+
"pliki_sha256": {
|
| 202 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 203 |
+
},
|
| 204 |
+
"acc": 0.345051,
|
| 205 |
+
"acc_ci95": [
|
| 206 |
+
0.335986,
|
| 207 |
+
0.354611
|
| 208 |
+
],
|
| 209 |
+
"acc_norm": 0.438259,
|
| 210 |
+
"acc_norm_ci95": [
|
| 211 |
+
0.428401,
|
| 212 |
+
0.447921
|
| 213 |
+
],
|
| 214 |
+
"split": "validation",
|
| 215 |
+
"correct_primary": 4401
|
| 216 |
+
},
|
| 217 |
+
"mmlu_pl": {
|
| 218 |
+
"n": 14042,
|
| 219 |
+
"n_pelne": 14042,
|
| 220 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 221 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 222 |
+
"pliki_sha256": {
|
| 223 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 224 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 225 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 226 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 227 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 228 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 229 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 230 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 231 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 232 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 233 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 234 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 235 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 236 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 237 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 238 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 239 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 240 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 241 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 242 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 243 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 244 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 245 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 246 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 247 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 248 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 249 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 250 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 251 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 252 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 253 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 254 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 255 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 256 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 257 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 258 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 259 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 260 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 261 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 262 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 263 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 264 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 265 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 266 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 267 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 268 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 269 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 270 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 271 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 272 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 273 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 274 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 275 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 276 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 277 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 278 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 279 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 280 |
+
},
|
| 281 |
+
"acc": 0.240137,
|
| 282 |
+
"acc_ci95": [
|
| 283 |
+
0.233015,
|
| 284 |
+
0.247116
|
| 285 |
+
],
|
| 286 |
+
"acc_norm": 0.240137,
|
| 287 |
+
"acc_norm_ci95": [
|
| 288 |
+
0.233158,
|
| 289 |
+
0.247187
|
| 290 |
+
],
|
| 291 |
+
"split": "test; micro-average over subjects",
|
| 292 |
+
"correct_primary": 3372
|
| 293 |
+
}
|
| 294 |
+
}
|
| 295 |
+
},
|
| 296 |
+
"1.08": {
|
| 297 |
+
"step": 1709,
|
| 298 |
+
"weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
|
| 299 |
+
"weights_published": true,
|
| 300 |
+
"source_results_sha256": "108e43d0ac726f946c0480ff00319eb1811a3e254f05b1540156b0db11d1566e",
|
| 301 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 302 |
+
"tasks": {
|
| 303 |
+
"arc_challenge": {
|
| 304 |
+
"n": 1172,
|
| 305 |
+
"n_pelne": 1172,
|
| 306 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 307 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 308 |
+
"pliki_sha256": {
|
| 309 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 310 |
+
},
|
| 311 |
+
"acc": 0.271331,
|
| 312 |
+
"acc_ci95": [
|
| 313 |
+
0.245734,
|
| 314 |
+
0.296928
|
| 315 |
+
],
|
| 316 |
+
"acc_norm": 0.300341,
|
| 317 |
+
"acc_norm_ci95": [
|
| 318 |
+
0.273891,
|
| 319 |
+
0.326792
|
| 320 |
+
],
|
| 321 |
+
"split": "test",
|
| 322 |
+
"correct_primary": 352
|
| 323 |
+
},
|
| 324 |
+
"hellaswag": {
|
| 325 |
+
"n": 10042,
|
| 326 |
+
"n_pelne": 10042,
|
| 327 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 328 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 329 |
+
"pliki_sha256": {
|
| 330 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 331 |
+
},
|
| 332 |
+
"acc": 0.350926,
|
| 333 |
+
"acc_ci95": [
|
| 334 |
+
0.341765,
|
| 335 |
+
0.360486
|
| 336 |
+
],
|
| 337 |
+
"acc_norm": 0.434176,
|
| 338 |
+
"acc_norm_ci95": [
|
| 339 |
+
0.424318,
|
| 340 |
+
0.443935
|
| 341 |
+
],
|
| 342 |
+
"split": "validation",
|
| 343 |
+
"correct_primary": 4360
|
| 344 |
+
},
|
| 345 |
+
"piqa": {
|
| 346 |
+
"n": 1838,
|
| 347 |
+
"n_pelne": 1838,
|
| 348 |
+
"dataset_repo": "baber/piqa",
|
| 349 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 350 |
+
"pliki_sha256": {
|
| 351 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 352 |
+
},
|
| 353 |
+
"acc": 0.681719,
|
| 354 |
+
"acc_ci95": [
|
| 355 |
+
0.661045,
|
| 356 |
+
0.702938
|
| 357 |
+
],
|
| 358 |
+
"acc_norm": 0.662133,
|
| 359 |
+
"acc_norm_ci95": [
|
| 360 |
+
0.641458,
|
| 361 |
+
0.683351
|
| 362 |
+
],
|
| 363 |
+
"split": "validation",
|
| 364 |
+
"correct_primary": 1217
|
| 365 |
+
},
|
| 366 |
+
"winogrande": {
|
| 367 |
+
"n": 1267,
|
| 368 |
+
"n_pelne": 1267,
|
| 369 |
+
"dataset_repo": "allenai/winogrande",
|
| 370 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 371 |
+
"pliki_sha256": {
|
| 372 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 373 |
+
},
|
| 374 |
+
"acc": 0.536701,
|
| 375 |
+
"acc_ci95": [
|
| 376 |
+
0.509077,
|
| 377 |
+
0.564325
|
| 378 |
+
],
|
| 379 |
+
"acc_norm": 0.536701,
|
| 380 |
+
"acc_norm_ci95": [
|
| 381 |
+
0.508287,
|
| 382 |
+
0.563536
|
| 383 |
+
],
|
| 384 |
+
"split": "validation",
|
| 385 |
+
"correct_primary": 680
|
| 386 |
+
},
|
| 387 |
+
"boolq": {
|
| 388 |
+
"n": 3270,
|
| 389 |
+
"n_pelne": 3270,
|
| 390 |
+
"dataset_repo": "aps/super_glue",
|
| 391 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 392 |
+
"pliki_sha256": {
|
| 393 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 394 |
+
},
|
| 395 |
+
"acc": 0.599694,
|
| 396 |
+
"acc_ci95": [
|
| 397 |
+
0.582569,
|
| 398 |
+
0.615902
|
| 399 |
+
],
|
| 400 |
+
"acc_norm": 0.620489,
|
| 401 |
+
"acc_norm_ci95": [
|
| 402 |
+
0.603364,
|
| 403 |
+
0.637309
|
| 404 |
+
],
|
| 405 |
+
"split": "validation",
|
| 406 |
+
"correct_primary": 1961
|
| 407 |
+
},
|
| 408 |
+
"openbookqa": {
|
| 409 |
+
"n": 500,
|
| 410 |
+
"n_pelne": 500,
|
| 411 |
+
"dataset_repo": "allenai/openbookqa",
|
| 412 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 413 |
+
"pliki_sha256": {
|
| 414 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 415 |
+
},
|
| 416 |
+
"acc": 0.23,
|
| 417 |
+
"acc_ci95": [
|
| 418 |
+
0.194,
|
| 419 |
+
0.266
|
| 420 |
+
],
|
| 421 |
+
"acc_norm": 0.348,
|
| 422 |
+
"acc_norm_ci95": [
|
| 423 |
+
0.306,
|
| 424 |
+
0.39
|
| 425 |
+
],
|
| 426 |
+
"split": "test",
|
| 427 |
+
"correct_primary": 174
|
| 428 |
+
},
|
| 429 |
+
"belebele_pol": {
|
| 430 |
+
"n": 900,
|
| 431 |
+
"n_pelne": 900,
|
| 432 |
+
"dataset_repo": "facebook/belebele",
|
| 433 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 434 |
+
"pliki_sha256": {
|
| 435 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 436 |
+
},
|
| 437 |
+
"acc": 0.24,
|
| 438 |
+
"acc_ci95": [
|
| 439 |
+
0.212222,
|
| 440 |
+
0.268889
|
| 441 |
+
],
|
| 442 |
+
"acc_norm": 0.24,
|
| 443 |
+
"acc_norm_ci95": [
|
| 444 |
+
0.212222,
|
| 445 |
+
0.267778
|
| 446 |
+
],
|
| 447 |
+
"split": "pol_Latn evaluation set",
|
| 448 |
+
"correct_primary": 216
|
| 449 |
+
},
|
| 450 |
+
"hellaswag_pl": {
|
| 451 |
+
"n": 10042,
|
| 452 |
+
"n_pelne": 10042,
|
| 453 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 454 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 455 |
+
"pliki_sha256": {
|
| 456 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 457 |
+
},
|
| 458 |
+
"acc": 0.352221,
|
| 459 |
+
"acc_ci95": [
|
| 460 |
+
0.342661,
|
| 461 |
+
0.361581
|
| 462 |
+
],
|
| 463 |
+
"acc_norm": 0.438956,
|
| 464 |
+
"acc_norm_ci95": [
|
| 465 |
+
0.428998,
|
| 466 |
+
0.448715
|
| 467 |
+
],
|
| 468 |
+
"split": "validation",
|
| 469 |
+
"correct_primary": 4408
|
| 470 |
+
},
|
| 471 |
+
"mmlu_pl": {
|
| 472 |
+
"n": 14042,
|
| 473 |
+
"n_pelne": 14042,
|
| 474 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 475 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 476 |
+
"pliki_sha256": {
|
| 477 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 478 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 479 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 480 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 481 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 482 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 483 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 484 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 485 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 486 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 487 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 488 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 489 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 490 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 491 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 492 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 493 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 494 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 495 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 496 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 497 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 498 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 499 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 500 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 501 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 502 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 503 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 504 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 505 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 506 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 507 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 508 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 509 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 510 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 511 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 512 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 513 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 514 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 515 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 516 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 517 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 518 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 519 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 520 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 521 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 522 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 523 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 524 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 525 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 526 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 527 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 528 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 529 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 530 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 531 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 532 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 533 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 534 |
+
},
|
| 535 |
+
"acc": 0.23038,
|
| 536 |
+
"acc_ci95": [
|
| 537 |
+
0.223401,
|
| 538 |
+
0.237431
|
| 539 |
+
],
|
| 540 |
+
"acc_norm": 0.23038,
|
| 541 |
+
"acc_norm_ci95": [
|
| 542 |
+
0.223259,
|
| 543 |
+
0.237288
|
| 544 |
+
],
|
| 545 |
+
"split": "test; micro-average over subjects",
|
| 546 |
+
"correct_primary": 3235
|
| 547 |
+
}
|
| 548 |
+
}
|
| 549 |
+
}
|
| 550 |
+
}
|
| 551 |
+
}
|
README.md
CHANGED
|
@@ -10,14 +10,13 @@ tags:
|
|
| 10 |
- causal-lm
|
| 11 |
- continued-pretraining
|
| 12 |
- experimental
|
| 13 |
-
- guard-failed
|
| 14 |
---
|
| 15 |
|
| 16 |
# Slayer1B 1.08
|
| 17 |
|
| 18 |
**Arkadiusz SΕota, Anna Sieczka, Piotr StyΕa / Fabryka AI (SlayerLab)**
|
| 19 |
|
| 20 |
-
**
|
| 21 |
|
| 22 |
[Public training trace](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a).
|
| 23 |
|
|
@@ -25,35 +24,6 @@ tags:
|
|
| 25 |
|
| 26 |
Train on as many approved token positions as the fixed compute budget permits, recording actual progress and expense at the spot prices observed during the run. Synthetic speed, real-data training and downstream evaluation are separate measurements. The expense report covers the billed session, including preparation and evaluation; steady-state cost per billion tokens is an extrapolation, not that bill.
|
| 27 |
|
| 28 |
-
## Pre-registered evaluation and outcome
|
| 29 |
-
|
| 30 |
-
FINAL and 1.07 were evaluated on the same B200 hardware. Reported 95% CIs use paired bootstrap with 10,000 draws; one-sided p-values use Holm correction over three confirmatory endpoints. One run means sampling uncertainty, not training-seed variance. Mathematics uses base mode, three-shot, strict pass@1; its baseline exceeds the prespecified 5% validity threshold. Domain bpb uses matched fixed samples of at most two million native tokens, scored per UTF-8 byte.
|
| 31 |
-
|
| 32 |
-
| Confirmatory endpoint | n | 1.07 | 1.08 | Difference [95% CI] | Holm p |
|
| 33 |
-
|---|---:|---:|---:|---|---:|
|
| 34 |
-
| Polish mathematics pass@1 | 300 items | 6.33% | 12.00% | +5.67 pp [+1.67, +9.67] | 0.0035 |
|
| 35 |
-
| Polish logic bpb | 314 documents | 0.78727 | 0.70895 | β0.07832 [β0.09000, β0.06865] | 0.0003 |
|
| 36 |
-
| Within-source English technical VALID bpb | 487 documents | 0.66661 | 0.38549 | β0.28112 [β0.34537, β0.23652] | 0.0003 |
|
| 37 |
-
|
| 38 |
-
All three endpoints pass their confirmatory tests. Technical VALID shares source repositories with TRAIN; only exact text duplicates were removed. Its gain is within training-source projects, **not evidence of generalization to unseen technical repositories**.
|
| 39 |
-
|
| 40 |
-
| Preregistered guard | n | 1.07 | 1.08 | Difference | Allowed difference | Outcome |
|
| 41 |
-
|---|---:|---:|---:|---:|---|---|
|
| 42 |
-
| ARC-Easy | 2,376 | 58.67% | 56.19% | β2.48 pp | β₯ β1.00 pp | **FAIL** |
|
| 43 |
-
| BLiMP | 67,000 | 77.90% | 76.91% | β0.99 pp | β₯ β1.00 pp | PASS, near boundary |
|
| 44 |
-
| ARC-Easy-PL | 2,376 | 46.93% | 47.14% | +0.21 pp | β₯ β1.00 pp | PASS |
|
| 45 |
-
| MultiBLiMP-PL | 3,272 | 98.84% | 98.47% | β0.37 pp | β₯ β1.00 pp | PASS |
|
| 46 |
-
| Polish H bpb | 1,056 documents | 0.75459 | 0.74835 | β0.00624 | β€ +0.005 | PASS |
|
| 47 |
-
|
| 48 |
-
The frozen rule makes **any guard failure prevent acceptance**; endpoint gains do not override it.
|
| 49 |
-
|
| 50 |
-
| Descriptive diagnostic, not confirmatory | n | 1.07 | 1.08 |
|
| 51 |
-
|---|---:|---:|---:|
|
| 52 |
-
| Held-out reasoning v2 pass@1 | 360 | 13.33% | 9.44% |
|
| 53 |
-
| Arithmetic grid pass@1 | 500 | 12.20% | 21.20% |
|
| 54 |
-
| WikiText-2 byte perplexity | 1,292,008 bytes | 2.1837 | 2.1939 |
|
| 55 |
-
| Polish H hit-free subset bpb | 711 documents | 0.78425 | 0.77805 |
|
| 56 |
-
|
| 57 |
## Model and starting point
|
| 58 |
|
| 59 |
The run initializes from [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md), revision `a090b54e658fb57dc5166c1fcc90cc6bf5c80106`. Its published card describes a Polish-dominant decoder-only model with **1,073,912,851 parameters**, a **32,768-token v6 vocabulary**, and final checkpoint **22,327**. These are base-model properties.
|
|
@@ -82,9 +52,10 @@ The continuation preserves that architecture and tokenizer. Its task is causal t
|
|
| 82 |
| Steady interval throughput | Approximately 572,000 positions/s; not positions divided by the whole training window |
|
| 83 |
| Training loss | Approximately 2.20 at update 10 to 1.99 at completion; not an evaluation score |
|
| 84 |
| Training GPU cost | Approximately USD 7.73, or USD 4.32 per billion processed positions, calculated from the observed window and spot rate |
|
| 85 |
-
|
|
|
|
|
| 86 |
|
| 87 |
-
Input capacity, processed positions, steady interval speed and the complete billed session remain separate quantities.
|
| 88 |
|
| 89 |
## Training inputs
|
| 90 |
|
|
@@ -102,6 +73,32 @@ A replay-only continued-pretraining control is not budgeted for the priority run
|
|
| 102 |
|
| 103 |
Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
|
| 104 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
## Licensing and limitations
|
| 106 |
|
| 107 |
Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.
|
|
|
|
| 10 |
- causal-lm
|
| 11 |
- continued-pretraining
|
| 12 |
- experimental
|
|
|
|
| 13 |
---
|
| 14 |
|
| 15 |
# Slayer1B 1.08
|
| 16 |
|
| 17 |
**Arkadiusz SΕota, Anna Sieczka, Piotr StyΕa / Fabryka AI (SlayerLab)**
|
| 18 |
|
| 19 |
+
**Slayer1B 1.08 is an experimental base language model continued from Slayer1B 1.07.** Training completed on 9 November 2026 after 1,709 updates and 1,792,016,384 processed token positions, using Polish analytical text, English technical text and replay. It is not instruction-tuned and is distinct from the earlier 1.08b ablation.
|
| 20 |
|
| 21 |
[Public training trace](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a).
|
| 22 |
|
|
|
|
| 24 |
|
| 25 |
Train on as many approved token positions as the fixed compute budget permits, recording actual progress and expense at the spot prices observed during the run. Synthetic speed, real-data training and downstream evaluation are separate measurements. The expense report covers the billed session, including preparation and evaluation; steady-state cost per billion tokens is an extrapolation, not that bill.
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
## Model and starting point
|
| 28 |
|
| 29 |
The run initializes from [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md), revision `a090b54e658fb57dc5166c1fcc90cc6bf5c80106`. Its published card describes a Polish-dominant decoder-only model with **1,073,912,851 parameters**, a **32,768-token v6 vocabulary**, and final checkpoint **22,327**. These are base-model properties.
|
|
|
|
| 52 |
| Steady interval throughput | Approximately 572,000 positions/s; not positions divided by the whole training window |
|
| 53 |
| Training loss | Approximately 2.20 at update 10 to 1.99 at completion; not an evaluation score |
|
| 54 |
| Training GPU cost | Approximately USD 7.73, or USD 4.32 per billion processed positions, calculated from the observed window and spot rate |
|
| 55 |
+
| Cost through evaluation completion | Approximately USD 21.5; preparation β11.2, training 7.73, evaluation/export β2.6. Provider-counter attribution, not a separate settled invoice |
|
| 56 |
+
| Complete allocation cost | Approximately USD 24.1; the allocation was preempted shortly afterwards and released, rather than reused for 1.09 |
|
| 57 |
|
| 58 |
+
Input capacity, processed positions, steady interval speed and the complete billed session remain separate quantities.
|
| 59 |
|
| 60 |
## Training inputs
|
| 61 |
|
|
|
|
| 73 |
|
| 74 |
Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
|
| 75 |
|
| 76 |
+
## Standard benchmark measurements
|
| 77 |
+
|
| 78 |
+
We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol work and evaluation proposals. The team's independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is a separate measurement source. **The table below reports SlayerLab evaluations**, also published in the [1.07 training record](https://track.fabryka.ai/run/4973c068-8dd5-59d8-b6c4-671ca3beb925) and the [1.08 training record](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a); it is not a copy of that independent run.
|
| 79 |
+
|
| 80 |
+
All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
|
| 81 |
+
|
| 82 |
+
| Task | Metric | n | 1.07, update 22,327 | 1.08, update 1,709 |
|
| 83 |
+
|---|---|---:|---:|---:|
|
| 84 |
+
| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] |
|
| 85 |
+
| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] |
|
| 86 |
+
| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] |
|
| 87 |
+
| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] |
|
| 88 |
+
| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] |
|
| 89 |
+
| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] |
|
| 90 |
+
| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] |
|
| 91 |
+
| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] |
|
| 92 |
+
| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] |
|
| 93 |
+
|
| 94 |
+
Intervals use the percentile bootstrap with 10,000 replicates, seed base 20261110 and a model-specific stream. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
|
| 95 |
+
|
| 96 |
+
[Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 97 |
+
|
| 98 |
+
**Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
|
| 99 |
+
|
| 100 |
+
Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
|
| 101 |
+
|
| 102 |
## Licensing and limitations
|
| 103 |
|
| 104 |
Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.
|