SlayerLab
/

Slayer1B-1.09

+{
+  "schema_version": 1,
+  "measurement_owner": "SlayerLab / Fabryka AI",
+  "status": "descriptive; training-lineage contamination scan pending",
+  "source_aggregate_sha256": "ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db",
+  "source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
+  "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
+  "community_report_is_separate_measurement": true,
+  "protocol": {
+    "few_shot": 0,
+    "dtype": "float32",
+    "context_limit": 4096,
+    "initial_token_id": 0,
+    "special_tokens": false,
+    "truncation": "left",
+    "scoring": "summed continuation-token log likelihood; task-specific partial scoring for WinoGrande",
+    "metric_units": "fraction; multiply by 100 for card percentages",
+    "primary_metrics": {
+      "arc_challenge": "acc_norm",
+      "hellaswag": "acc_norm",
+      "piqa": "acc_norm",
+      "winogrande": "acc",
+      "boolq": "acc",
+      "openbookqa": "acc_norm",
+      "belebele_pol": "acc",
+      "hellaswag_pl": "acc_norm",
+      "mmlu_pl": "acc"
+    },
+    "mmlu_aggregation": "micro-average over questions",
+    "polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b"
+  },
+  "ci": {
+    "level": 0.95,
+    "method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
+    "replicates": 10000,
+    "base_seed": 20261110,
+    "seed_scope": "base seed plus model label, as source generator",
+    "assumption": "items treated as independent; no subject, template or source clusters",
+    "interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
+  },
+  "models": {
+    "1.07": {
+      "step": 22327,
+      "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
+      "weights_published": true,
+      "source_results_sha256": "9d89a9cdc1ea66723c9c709930d987900609bee84de5598dba400231407babf2",
+      "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
+      "tasks": {
+        "arc_challenge": {
+          "n": 1172,
+          "n_pelne": 1172,
+          "dataset_repo": "allenai/ai2_arc",
+          "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
+          "pliki_sha256": {
+            "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
+          },
+          "acc": 0.261945,
+          "acc_ci95": [
+            0.237201,
+            0.286689
+          ],
+          "acc_norm": 0.273038,
+          "acc_norm_ci95": [
+            0.24744,
+            0.298635
+          ],
+          "split": "test",
+          "correct_primary": 320
+        },
+        "hellaswag": {
+          "n": 10042,
+          "n_pelne": 10042,
+          "dataset_repo": "Rowan/hellaswag",
+          "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
+          "pliki_sha256": {
+            "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
+          },
+          "acc": 0.351324,
+          "acc_ci95": [
+            0.341762,
+            0.360486
+          ],
+          "acc_norm": 0.433778,
+          "acc_norm_ci95": [
+            0.424318,
+            0.443438
+          ],
+          "split": "validation",
+          "correct_primary": 4356
+        },
+        "piqa": {
+          "n": 1838,
+          "n_pelne": 1838,
+          "dataset_repo": "baber/piqa",
+          "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
+          "pliki_sha256": {
+            "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
+          },
+          "acc": 0.670294,
+          "acc_ci95": [
+            0.648531,
+            0.691526
+          ],
+          "acc_norm": 0.66975,
+          "acc_norm_ci95": [
+            0.647987,
+            0.691513
+          ],
+          "split": "validation",
+          "correct_primary": 1231
+        },
+        "winogrande": {
+          "n": 1267,
+          "n_pelne": 1267,
+          "dataset_repo": "allenai/winogrande",
+          "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
+          "pliki_sha256": {
+            "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
+          },
+          "acc": 0.535912,
+          "acc_ci95": [
+            0.508287,
+            0.563536
+          ],
+          "acc_norm": 0.535912,
+          "acc_norm_ci95": [
+            0.509077,
+            0.563536
+          ],
+          "split": "validation",
+          "correct_primary": 679
+        },
+        "boolq": {
+          "n": 3270,
+          "n_pelne": 3270,
+          "dataset_repo": "aps/super_glue",
+          "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
+          "pliki_sha256": {
+            "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
+          },
+          "acc": 0.582875,
+          "acc_ci95": [
+            0.566055,
+            0.599694
+          ],
+          "acc_norm": 0.621101,
+          "acc_norm_ci95": [
+            0.604281,
+            0.637615
+          ],
+          "split": "validation",
+          "correct_primary": 1906
+        },
+        "openbookqa": {
+          "n": 500,
+          "n_pelne": 500,
+          "dataset_repo": "allenai/openbookqa",
+          "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
+          "pliki_sha256": {
+            "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
+          },
+          "acc": 0.2,
+          "acc_ci95": [
+            0.166,
+            0.236
+          ],
+          "acc_norm": 0.336,
+          "acc_norm_ci95": [
+            0.294,
+            0.378
+          ],
+          "split": "test",
+          "correct_primary": 168
+        },
+        "belebele_pol": {
+          "n": 900,
+          "n_pelne": 900,
+          "dataset_repo": "facebook/belebele",
+          "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
+          "pliki_sha256": {
+            "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
+          },
+          "acc": 0.26,
+          "acc_ci95": [
+            0.231111,
+            0.288889
+          ],
+          "acc_norm": 0.26,
+          "acc_norm_ci95": [
+            0.232222,
+            0.288889
+          ],
+          "split": "pol_Latn evaluation set",
+          "correct_primary": 234
+        },
+        "hellaswag_pl": {
+          "n": 10042,
+          "n_pelne": 10042,
+          "dataset_repo": "Eurolingua/hellaswagx",
+          "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
+          "pliki_sha256": {
+            "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
+          },
+          "acc": 0.345051,
+          "acc_ci95": [
+            0.335986,
+            0.354611
+          ],
+          "acc_norm": 0.438259,
+          "acc_norm_ci95": [
+            0.428401,
+            0.447921
+          ],
+          "split": "validation",
+          "correct_primary": 4401
+        },
+        "mmlu_pl": {
+          "n": 14042,
+          "n_pelne": 14042,
+          "dataset_repo": "Eurolingua/mmlux",
+          "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
+          "pliki_sha256": {
+            "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
+            "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
+            "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
+            "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
+            "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
+            "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
+            "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
+            "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
+            "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
+            "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
+            "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
+            "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
+            "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
+            "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
+            "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
+            "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
+            "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
+            "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
+            "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
+            "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
+            "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
+            "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
+            "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
+            "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
+            "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
+            "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
+            "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
+            "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
+            "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
+            "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
+            "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
+            "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
+            "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
+            "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
+            "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
+            "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
+            "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
+            "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
+            "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
+            "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
+            "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
+            "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
+            "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
+            "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
+            "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
+            "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
+            "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
+            "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
+            "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
+            "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
+            "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
+            "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
+            "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
+            "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
+            "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
+            "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
+            "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
+          },
+          "acc": 0.240137,
+          "acc_ci95": [
+            0.233015,
+            0.247116
+          ],
+          "acc_norm": 0.240137,
+          "acc_norm_ci95": [
+            0.233158,
+            0.247187
+          ],
+          "split": "test; micro-average over subjects",
+          "correct_primary": 3372
+        }
+      }
+    },
+    "1.08": {
+      "step": 1709,
+      "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
+      "weights_published": true,
+      "source_results_sha256": "108e43d0ac726f946c0480ff00319eb1811a3e254f05b1540156b0db11d1566e",
+      "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
+      "tasks": {
+        "arc_challenge": {
+          "n": 1172,
+          "n_pelne": 1172,
+          "dataset_repo": "allenai/ai2_arc",
+          "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
+          "pliki_sha256": {
+            "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
+          },
+          "acc": 0.271331,
+          "acc_ci95": [
+            0.245734,
+            0.296928
+          ],
+          "acc_norm": 0.300341,
+          "acc_norm_ci95": [
+            0.273891,
+            0.326792
+          ],
+          "split": "test",
+          "correct_primary": 352
+        },
+        "hellaswag": {
+          "n": 10042,
+          "n_pelne": 10042,
+          "dataset_repo": "Rowan/hellaswag",
+          "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
+          "pliki_sha256": {
+            "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
+          },
+          "acc": 0.350926,
+          "acc_ci95": [
+            0.341765,
+            0.360486
+          ],
+          "acc_norm": 0.434176,
+          "acc_norm_ci95": [
+            0.424318,
+            0.443935
+          ],
+          "split": "validation",
+          "correct_primary": 4360
+        },
+        "piqa": {
+          "n": 1838,
+          "n_pelne": 1838,
+          "dataset_repo": "baber/piqa",
+          "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
+          "pliki_sha256": {
+            "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
+          },
+          "acc": 0.681719,
+          "acc_ci95": [
+            0.661045,
+            0.702938
+          ],
+          "acc_norm": 0.662133,
+          "acc_norm_ci95": [
+            0.641458,
+            0.683351
+          ],
+          "split": "validation",
+          "correct_primary": 1217
+        },
+        "winogrande": {
+          "n": 1267,
+          "n_pelne": 1267,
+          "dataset_repo": "allenai/winogrande",
+          "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
+          "pliki_sha256": {
+            "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
+          },
+          "acc": 0.536701,
+          "acc_ci95": [
+            0.509077,
+            0.564325
+          ],
+          "acc_norm": 0.536701,
+          "acc_norm_ci95": [
+            0.508287,
+            0.563536
+          ],
+          "split": "validation",
+          "correct_primary": 680
+        },
+        "boolq": {
+          "n": 3270,
+          "n_pelne": 3270,
+          "dataset_repo": "aps/super_glue",
+          "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
+          "pliki_sha256": {
+            "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
+          },
+          "acc": 0.599694,
+          "acc_ci95": [
+            0.582569,
+            0.615902
+          ],
+          "acc_norm": 0.620489,
+          "acc_norm_ci95": [
+            0.603364,
+            0.637309
+          ],
+          "split": "validation",
+          "correct_primary": 1961
+        },
+        "openbookqa": {
+          "n": 500,
+          "n_pelne": 500,
+          "dataset_repo": "allenai/openbookqa",
+          "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
+          "pliki_sha256": {
+            "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
+          },
+          "acc": 0.23,
+          "acc_ci95": [
+            0.194,
+            0.266
+          ],
+          "acc_norm": 0.348,
+          "acc_norm_ci95": [
+            0.306,
+            0.39
+          ],
+          "split": "test",
+          "correct_primary": 174
+        },
+        "belebele_pol": {
+          "n": 900,
+          "n_pelne": 900,
+          "dataset_repo": "facebook/belebele",
+          "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
+          "pliki_sha256": {
+            "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
+          },
+          "acc": 0.24,
+          "acc_ci95": [
+            0.212222,
+            0.268889
+          ],
+          "acc_norm": 0.24,
+          "acc_norm_ci95": [
+            0.212222,
+            0.267778
+          ],
+          "split": "pol_Latn evaluation set",
+          "correct_primary": 216
+        },
+        "hellaswag_pl": {
+          "n": 10042,
+          "n_pelne": 10042,
+          "dataset_repo": "Eurolingua/hellaswagx",
+          "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
+          "pliki_sha256": {
+            "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
+          },
+          "acc": 0.352221,
+          "acc_ci95": [
+            0.342661,
+            0.361581
+          ],
+          "acc_norm": 0.438956,
+          "acc_norm_ci95": [
+            0.428998,
+            0.448715
+          ],
+          "split": "validation",
+          "correct_primary": 4408
+        },
+        "mmlu_pl": {
+          "n": 14042,
+          "n_pelne": 14042,
+          "dataset_repo": "Eurolingua/mmlux",
+          "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
+          "pliki_sha256": {
+            "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
+            "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
+            "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
+            "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
+            "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
+            "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
+            "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
+            "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
+            "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
+            "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
+            "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
+            "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
+            "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
+            "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
+            "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
+            "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
+            "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
+            "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
+            "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
+            "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
+            "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
+            "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
+            "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
+            "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
+            "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
+            "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
+            "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
+            "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
+            "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
+            "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
+            "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
+            "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
+            "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
+            "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
+            "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
+            "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
+            "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
+            "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
+            "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
+            "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
+            "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
+            "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
+            "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
+            "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
+            "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
+            "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
+            "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
+            "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
+            "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
+            "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
+            "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
+            "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
+            "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
+            "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
+            "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
+            "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
+            "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
+          },
+          "acc": 0.23038,
+          "acc_ci95": [
+            0.223401,
+            0.237431
+          ],
+          "acc_norm": 0.23038,
+          "acc_norm_ci95": [
+            0.223259,
+            0.237288
+          ],
+          "split": "test; micro-average over subjects",
+          "correct_primary": 3235
+        }
+      }
+    },
+    "1.09-k5000": {
+      "step": 5000,
+      "weights_sha256": "ccd7fc17b834d1de82ac53d0252085cbab567ac96676d0e263cd463a05745631",
+      "weights_published": false,
+      "source_results_sha256": "ec8d493463d99687f85a06654cef7706b3d37f0c9b9c3e760512f2baac34d632",
+      "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
+      "tasks": {
+        "arc_challenge": {
+          "n": 1172,
+          "n_pelne": 1172,
+          "dataset_repo": "allenai/ai2_arc",
+          "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
+          "pliki_sha256": {
+            "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
+          },
+          "acc": 0.288396,
+          "acc_ci95": [
+            0.262799,
+            0.314846
+          ],
+          "acc_norm": 0.293515,
+          "acc_norm_ci95": [
+            0.267918,
+            0.319966
+          ],
+          "split": "test",
+          "correct_primary": 344
+        },
+        "hellaswag": {
+          "n": 10042,
+          "n_pelne": 10042,
+          "dataset_repo": "Rowan/hellaswag",
+          "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
+          "pliki_sha256": {
+            "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
+          },
+          "acc": 0.370942,
+          "acc_ci95": [
+            0.361781,
+            0.380303
+          ],
+          "acc_norm": 0.465346,
+          "acc_norm_ci95": [
+            0.455487,
+            0.475204
+          ],
+          "split": "validation",
+          "correct_primary": 4673
+        },
+        "piqa": {
+          "n": 1838,
+          "n_pelne": 1838,
+          "dataset_repo": "baber/piqa",
+          "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
+          "pliki_sha256": {
+            "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
+          },
+          "acc": 0.688248,
+          "acc_ci95": [
+            0.667029,
+            0.708923
+          ],
+          "acc_norm": 0.682807,
+          "acc_norm_ci95": [
+            0.662119,
+            0.704026
+          ],
+          "split": "validation",
+          "correct_primary": 1255
+        },
+        "winogrande": {
+          "n": 1267,
+          "n_pelne": 1267,
+          "dataset_repo": "allenai/winogrande",
+          "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
+          "pliki_sha256": {
+            "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
+          },
+          "acc": 0.549329,
+          "acc_ci95": [
+            0.521705,
+            0.576953
+          ],
+          "acc_norm": 0.549329,
+          "acc_norm_ci95": [
+            0.521705,
+            0.576953
+          ],
+          "split": "validation",
+          "correct_primary": 696
+        },
+        "boolq": {
+          "n": 3270,
+          "n_pelne": 3270,
+          "dataset_repo": "aps/super_glue",
+          "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
+          "pliki_sha256": {
+            "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
+          },
+          "acc": 0.618043,
+          "acc_ci95": [
+            0.601827,
+            0.634862
+          ],
+          "acc_norm": 0.623242,
+          "acc_norm_ci95": [
+            0.606728,
+            0.640061
+          ],
+          "split": "validation",
+          "correct_primary": 2021
+        },
+        "openbookqa": {
+          "n": 500,
+          "n_pelne": 500,
+          "dataset_repo": "allenai/openbookqa",
+          "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
+          "pliki_sha256": {
+            "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
+          },
+          "acc": 0.252,
+          "acc_ci95": [
+            0.216,
+            0.29
+          ],
+          "acc_norm": 0.356,
+          "acc_norm_ci95": [
+            0.314,
+            0.4
+          ],
+          "split": "test",
+          "correct_primary": 178
+        },
+        "belebele_pol": {
+          "n": 900,
+          "n_pelne": 900,
+          "dataset_repo": "facebook/belebele",
+          "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
+          "pliki_sha256": {
+            "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
+          },
+          "acc": 0.241111,
+          "acc_ci95": [
+            0.213333,
+            0.268889
+          ],
+          "acc_norm": 0.241111,
+          "acc_norm_ci95": [
+            0.213333,
+            0.27
+          ],
+          "split": "pol_Latn evaluation set",
+          "correct_primary": 217
+        },
+        "hellaswag_pl": {
+          "n": 10042,
+          "n_pelne": 10042,
+          "dataset_repo": "Eurolingua/hellaswagx",
+          "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
+          "pliki_sha256": {
+            "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
+          },
+          "acc": 0.349632,
+          "acc_ci95": [
+            0.340271,
+            0.358992
+          ],
+          "acc_norm": 0.447321,
+          "acc_norm_ci95": [
+            0.437562,
+            0.45708
+          ],
+          "split": "validation",
+          "correct_primary": 4492
+        },
+        "mmlu_pl": {
+          "n": 14042,
+          "n_pelne": 14042,
+          "dataset_repo": "Eurolingua/mmlux",
+          "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
+          "pliki_sha256": {
+            "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
+            "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
+            "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
+            "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
+            "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
+            "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
+            "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
+            "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
+            "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
+            "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
+            "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
+            "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
+            "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
+            "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
+            "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
+            "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
+            "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
+            "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
+            "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
+            "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
+            "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
+            "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
+            "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
+            "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
+            "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
+            "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
+            "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
+            "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
+            "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
+            "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
+            "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
+            "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
+            "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
+            "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
+            "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
+            "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
+            "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
+            "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
+            "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
+            "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
+            "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
+            "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
+            "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
+            "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
+            "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
+            "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
+            "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
+            "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
+            "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
+            "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
+            "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
+            "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
+            "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
+            "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
+            "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
+            "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
+            "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
+          },
+          "acc": 0.240991,
+          "acc_ci95": [
+            0.23387,
+            0.248184
+          ],
+          "acc_norm": 0.240991,
+          "acc_norm_ci95": [
+            0.23387,
+            0.248042
+          ],
+          "split": "test; micro-average over subjects",
+          "correct_primary": 3384
+        }
+      }
+    }
+  }
+}

README.md CHANGED Viewed

@@ -9,14 +9,14 @@ tags:
 - causal-lm
 - pretraining
 - continued-pretraining
-- training-paused
 ---
 # Slayer1B 1.09
 **Arkadiusz Słota, Anna Sieczka, Piotr Styła / Fabryka AI (SlayerLab)**
-**Execution snapshot, 10 November 2026, 19:23 UTC: continued pretraining from Slayer1B 1.08 is paused by the cost guard after the on-node update-11,600 checkpoint. The objective remains useful training exposure within a total USD 425 campaign budget.** The operator stopped GPU allocation after four price readings exceeded the configured cost-per-token ceiling; the VM and disk are retained for recovery. This is not a completed model. The budget-priority decision replaces the earlier commitment to finish 80,000 updates and exceed 100B positions. Final length remains subject to measured cost and completion of learning-rate decay; roughly 35,000–40,000 updates is a price-dependent projection, not a selected or completed trainer length.
 | Verified execution evidence | Scope |
 |---|---|
@@ -30,13 +30,13 @@ The [CPT Track history](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea
 ## Objective and starting point
-The current objective is **the largest useful training exposure affordable within USD 425**, using eligible, diverse data filtered against the available ancestral references. The cost scenarios reviewed on 10 November project roughly **35,000–40,000 updates / 36.70B–41.94B new processed positions**, depending on spot prices, interruptions and operating costs. Final length must be rounded down to an affordable 5,000-update boundary, including the cost of completing decay. It does not promise 80,000 updates, more than 100B globally unique tokens or compliance with the earlier completion-date estimate. Model quality is evaluated separately; token positions per dollar measure operating efficiency, not a quality score.
 The starting weights are **[SlayerLab/Slayer1B-1.08](https://huggingface.co/SlayerLab/Slayer1B-1.08/tree/5f854a0cc7160594d54d1788d37a44e8ebabe16c)**, not random initialization or the abandoned attempt's checkpoint. Operators verified step-zero `INIT-FROM`, fresh optimizer state, approved c01 bytes, fresh numerical control and successful preflight before the observed updates. The selected public v5 launcher uses a separate output history for this continuation.
 The unchanged architecture is specified as **1,073,912,851 parameters**, 20 layers, width 2,048, 16 heads, SwiGLU, RMSNorm, RoPE base 500,000, QK normalization and value residuals. The reused v6 tokenizer has vocabulary 32,768 and pin `0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd`.
-Architecture and tokenizer lineage includes [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md). The selected 1.08 base was published as an experiment with an **ARC-Easy retention-guard FAIL**; choosing it for further training does not erase that verdict or establish a successful 1.09.
 The [training configuration](TRAINING_CONFIGURATION.md) distinguishes the still-configured 80,000-update scheduling argument from the budget-limited final length. Completed 1.08 and the abandoned first attempt remain separate historical evidence.
@@ -100,13 +100,31 @@ The PL/EN balance will follow the approved fresh-data inventory. The user explic
 New material requires licensing, PII filtering, contamination scans, grouped held-outs and deduplication against the base model's full data ancestry. A 1% document holdout is not necessarily 1% of tokens. Historical exposure in the starting weights cannot be undone by filtering new inputs; retrospective benchmark hits must be handled explicitly in evaluation denominators.
-## Evaluation and acceptance
-Descriptive checkpoint-5,000 evaluations are published in the CPT Track history, alongside 1.07 and 1.08 reference measurements. These aggregates have their own protocol and checkpoint identity; they do not constitute final model acceptance. Final acceptance criteria must be fixed before interpreting the eventual budget-limited endpoint.
-New benchmarks remain **descriptive, contamination check pending** until the retrospective checks are complete. No old threshold, early training-loss decrease or retrospectively chosen checkpoint is presented here as acceptance of the revised continuation.
-Comparison is descriptive: no replay-only control is budgeted, and initialization, exposure and schedule differ. No causal dataset benefit or multi-seed significance can be claimed.
 ## Cost, licensing and limitations

 - causal-lm
 - pretraining
 - continued-pretraining
+- training-in-progress
 ---
 # Slayer1B 1.09
 **Arkadiusz Słota, Anna Sieczka, Piotr Styła / Fabryka AI (SlayerLab)**
+**Execution snapshot, 10 November 2026, 21:35 UTC: continued pretraining from Slayer1B 1.08 has resumed on eight H200 GPUs. The objective remains useful training exposure within a total USD 425 campaign budget.** After the earlier cost pause, operators verified fresh numerical control, preflight and `RESUME @ 11600` at 21:28 UTC; the update-11,610 loss matched the pre-stop value. The public trace subsequently reached update 11,690. The tracking plan is 40,000 updates, while the trainer still retains its 80,000-update scheduling argument during the constant-rate phase. Neither value is a promise of completed exposure; final length depends on the remaining budget and completion of learning-rate decay. This is not a completed model or weight release.
 | Verified execution evidence | Scope |
 |---|---|
 ## Objective and starting point
+The current objective is **the largest useful training exposure affordable within USD 425**, using eligible, diverse data filtered against the available ancestral references. The public tracking plan is **40,000 updates**, not a selected final trainer length: lower spot prices can permit more updates, while interruptions and other costs can reduce the affordable total. Final length must be rounded down to an affordable 5,000-update boundary before WSD decay, including the cost of completing that decay. There is no promise of 80,000 updates, more than 100B globally unique tokens or the earlier completion date. Model quality is evaluated separately; positions per dollar measure operating efficiency, not a quality score.
 The starting weights are **[SlayerLab/Slayer1B-1.08](https://huggingface.co/SlayerLab/Slayer1B-1.08/tree/5f854a0cc7160594d54d1788d37a44e8ebabe16c)**, not random initialization or the abandoned attempt's checkpoint. Operators verified step-zero `INIT-FROM`, fresh optimizer state, approved c01 bytes, fresh numerical control and successful preflight before the observed updates. The selected public v5 launcher uses a separate output history for this continuation.
 The unchanged architecture is specified as **1,073,912,851 parameters**, 20 layers, width 2,048, 16 heads, SwiGLU, RMSNorm, RoPE base 500,000, QK normalization and value residuals. The reused v6 tokenizer has vocabulary 32,768 and pin `0640d3bd3674d7a6e59c540d945a2ad275a12e1bcaeb6007c37a90751f9815dd`.
+Architecture and tokenizer lineage includes [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md). The selected 1.08 base and the separate historical 1.08b ablation are different checkpoints; the continuation described here starts from 1.08.
 The [training configuration](TRAINING_CONFIGURATION.md) distinguishes the still-configured 80,000-update scheduling argument from the budget-limited final length. Completed 1.08 and the abandoned first attempt remain separate historical evidence.
 New material requires licensing, PII filtering, contamination scans, grouped held-outs and deduplication against the base model's full data ancestry. A 1% document holdout is not necessarily 1% of tokens. Historical exposure in the starting weights cannot be undone by filtering new inputs; retrospective benchmark hits must be handled explicitly in evaluation denominators.
+## Standard benchmark measurements — update 5,000
+We thank **BenchTeam / Fabryka AI** for benchmark implementations, protocol work and evaluation proposals. Its independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is distinct from the **SlayerLab measurements below**, published in the [CPT Track history](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a). Update 5,000 is an intermediate checkpoint, not the eventual budget-limited endpoint or an inference-ready weight release.
+The same nine full splits were scored zero-shot in float32 for 1.07 update 22,327, 1.08 update 1,709 and 1.09 update 5,000. Scores are percentages with **pointwise 95% confidence intervals**; the named metric is accuracy (`acc`) or length-normalized accuracy (`acc_norm`). WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
+| Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 |
+|---|---|---:|---:|---:|---:|
+| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] |
+| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] |
+| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] |
+| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] |
+| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] |
+| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] |
+| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] |
+| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] |
+| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] |
+Intervals use the percentile bootstrap with 10,000 replicates, seed base 20261110 and a model-specific stream. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired differences or training-seed variance; overlap is not a test of a difference.
+The [machine-readable aggregate record](EVALUATION.json) supplies exact values, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection, without prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
+**Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL are near their four-choice 25% random baseline. No overall average, ranking or release threshold is inferred. Numbers from different evaluators must not be silently substituted: prompting, BOS handling, normalization and software versions can change the result.
+There is no replay-only control for this continuation, and initialization, exposure and schedule differ between checkpoints. These measurements do not isolate a causal effect of the new datasets or quantify training-seed variance. Earlier internal diagnostics are separate historical records, not a criterion for naming or publishing this model.
 ## Cost, licensing and limitations