Maggio33 commited on
Commit
ea0945b
·
verified ·
1 Parent(s): a64b838

Slayer1B 1.08: update cards and documentation

Browse files
Files changed (2) hide show
  1. EVALUATION.json +509 -1
  2. README.md +14 -14
EVALUATION.json CHANGED
@@ -2,7 +2,7 @@
2
  "schema_version": 1,
3
  "measurement_owner": "SlayerLab / Fabryka AI",
4
  "status": "descriptive; training-lineage contamination scan pending",
5
- "source_aggregate_sha256": "fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796",
6
  "source_generator_sha256": null,
7
  "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
8
  "community_report_is_separate_measurement": true,
@@ -1573,6 +1573,514 @@
1573
  "correct_primary": 3351
1574
  }
1575
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1576
  }
1577
  },
1578
  "additional_measurements": {
 
2
  "schema_version": 1,
3
  "measurement_owner": "SlayerLab / Fabryka AI",
4
  "status": "descriptive; training-lineage contamination scan pending",
5
+ "source_aggregate_sha256": "2d257a95523605b06aa964cb974f65d86ab025875cc1f87dc98b242b90729440",
6
  "source_generator_sha256": null,
7
  "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
8
  "community_report_is_separate_measurement": true,
 
1573
  "correct_primary": 3351
1574
  }
1575
  }
1576
+ },
1577
+ "1.09-k25000": {
1578
+ "step": 25000,
1579
+ "weights_published": false,
1580
+ "weights_sha256": "46dfedda0d855f36f86fb1a0c0661809c654422c76bd1b3b2259e081f115b310",
1581
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
1582
+ "source_results_sha256": "7b1ac4f7710b11b64542eb193a2820144c3ba6482901159b20356d2928e6f9c5",
1583
+ "tasks": {
1584
+ "arc_challenge": {
1585
+ "n": 1172,
1586
+ "n_pelne": 1172,
1587
+ "dataset_repo": "allenai/ai2_arc",
1588
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
1589
+ "pliki_sha256": {
1590
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
1591
+ },
1592
+ "acc": 0.313993,
1593
+ "acc_ci95": [
1594
+ 0.288396,
1595
+ 0.340444
1596
+ ],
1597
+ "acc_norm": 0.337884,
1598
+ "acc_norm_ci95": [
1599
+ 0.31058,
1600
+ 0.365188
1601
+ ],
1602
+ "split": "test",
1603
+ "correct_primary": 396
1604
+ },
1605
+ "hellaswag": {
1606
+ "n": 10042,
1607
+ "n_pelne": 10042,
1608
+ "dataset_repo": "Rowan/hellaswag",
1609
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
1610
+ "pliki_sha256": {
1611
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
1612
+ },
1613
+ "acc": 0.397132,
1614
+ "acc_ci95": [
1615
+ 0.387373,
1616
+ 0.406592
1617
+ ],
1618
+ "acc_norm": 0.511651,
1619
+ "acc_norm_ci95": [
1620
+ 0.501792,
1621
+ 0.52141
1622
+ ],
1623
+ "split": "validation",
1624
+ "correct_primary": 5138
1625
+ },
1626
+ "piqa": {
1627
+ "n": 1838,
1628
+ "n_pelne": 1838,
1629
+ "dataset_repo": "baber/piqa",
1630
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
1631
+ "pliki_sha256": {
1632
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
1633
+ },
1634
+ "acc": 0.706202,
1635
+ "acc_ci95": [
1636
+ 0.685528,
1637
+ 0.726877
1638
+ ],
1639
+ "acc_norm": 0.715452,
1640
+ "acc_norm_ci95": [
1641
+ 0.694763,
1642
+ 0.736126
1643
+ ],
1644
+ "split": "validation",
1645
+ "correct_primary": 1315
1646
+ },
1647
+ "winogrande": {
1648
+ "n": 1267,
1649
+ "n_pelne": 1267,
1650
+ "dataset_repo": "allenai/winogrande",
1651
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
1652
+ "pliki_sha256": {
1653
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
1654
+ },
1655
+ "acc": 0.5588,
1656
+ "acc_ci95": [
1657
+ 0.531176,
1658
+ 0.586425
1659
+ ],
1660
+ "acc_norm": 0.5588,
1661
+ "acc_norm_ci95": [
1662
+ 0.531965,
1663
+ 0.586425
1664
+ ],
1665
+ "split": "validation",
1666
+ "correct_primary": 708
1667
+ },
1668
+ "boolq": {
1669
+ "n": 3270,
1670
+ "n_pelne": 3270,
1671
+ "dataset_repo": "aps/super_glue",
1672
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
1673
+ "pliki_sha256": {
1674
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
1675
+ },
1676
+ "acc": 0.522018,
1677
+ "acc_ci95": [
1678
+ 0.505191,
1679
+ 0.539144
1680
+ ],
1681
+ "acc_norm": 0.614679,
1682
+ "acc_norm_ci95": [
1683
+ 0.598165,
1684
+ 0.631498
1685
+ ],
1686
+ "split": "validation",
1687
+ "correct_primary": 1707
1688
+ },
1689
+ "openbookqa": {
1690
+ "n": 500,
1691
+ "n_pelne": 500,
1692
+ "dataset_repo": "allenai/openbookqa",
1693
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
1694
+ "pliki_sha256": {
1695
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
1696
+ },
1697
+ "acc": 0.26,
1698
+ "acc_ci95": [
1699
+ 0.22,
1700
+ 0.3
1701
+ ],
1702
+ "acc_norm": 0.372,
1703
+ "acc_norm_ci95": [
1704
+ 0.328,
1705
+ 0.414
1706
+ ],
1707
+ "split": "test",
1708
+ "correct_primary": 186
1709
+ },
1710
+ "belebele_pol": {
1711
+ "n": 900,
1712
+ "n_pelne": 900,
1713
+ "dataset_repo": "facebook/belebele",
1714
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
1715
+ "pliki_sha256": {
1716
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
1717
+ },
1718
+ "acc": 0.253333,
1719
+ "acc_ci95": [
1720
+ 0.224444,
1721
+ 0.282222
1722
+ ],
1723
+ "acc_norm": 0.253333,
1724
+ "acc_norm_ci95": [
1725
+ 0.224444,
1726
+ 0.282222
1727
+ ],
1728
+ "split": "pol_Latn evaluation set",
1729
+ "correct_primary": 228
1730
+ },
1731
+ "hellaswag_pl": {
1732
+ "n": 10042,
1733
+ "n_pelne": 10042,
1734
+ "dataset_repo": "Eurolingua/hellaswagx",
1735
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
1736
+ "pliki_sha256": {
1737
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
1738
+ },
1739
+ "acc": 0.342561,
1740
+ "acc_ci95": [
1741
+ 0.333499,
1742
+ 0.351524
1743
+ ],
1744
+ "acc_norm": 0.439355,
1745
+ "acc_norm_ci95": [
1746
+ 0.429397,
1747
+ 0.449114
1748
+ ],
1749
+ "split": "validation",
1750
+ "correct_primary": 4412
1751
+ },
1752
+ "mmlu_pl": {
1753
+ "n": 14042,
1754
+ "n_pelne": 14042,
1755
+ "dataset_repo": "Eurolingua/mmlux",
1756
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
1757
+ "pliki_sha256": {
1758
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
1759
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
1760
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
1761
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
1762
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
1763
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
1764
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
1765
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
1766
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
1767
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
1768
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
1769
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
1770
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
1771
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
1772
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
1773
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
1774
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
1775
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
1776
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
1777
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
1778
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
1779
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
1780
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
1781
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
1782
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
1783
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
1784
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
1785
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
1786
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
1787
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
1788
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
1789
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
1790
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
1791
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
1792
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
1793
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
1794
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
1795
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
1796
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
1797
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
1798
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
1799
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
1800
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
1801
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
1802
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
1803
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
1804
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
1805
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
1806
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
1807
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
1808
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
1809
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
1810
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
1811
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
1812
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
1813
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
1814
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
1815
+ },
1816
+ "acc": 0.242629,
1817
+ "acc_ci95": [
1818
+ 0.235721,
1819
+ 0.249608
1820
+ ],
1821
+ "acc_norm": 0.242629,
1822
+ "acc_norm_ci95": [
1823
+ 0.235579,
1824
+ 0.24968
1825
+ ],
1826
+ "split": "test; micro-average over subjects",
1827
+ "correct_primary": 3407
1828
+ }
1829
+ }
1830
+ },
1831
+ "1.09-k30000": {
1832
+ "step": 30000,
1833
+ "weights_published": false,
1834
+ "weights_sha256": "8a80bb721f6e98677fedcc4ea850896b3f3e5fe158b9f30915e7558bfa1c9764",
1835
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
1836
+ "source_results_sha256": "724d1fa367dcc57d07d4f6959a1cec40b98ddc6350179fc55248586fa1da6a3e",
1837
+ "tasks": {
1838
+ "arc_challenge": {
1839
+ "n": 1172,
1840
+ "n_pelne": 1172,
1841
+ "dataset_repo": "allenai/ai2_arc",
1842
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
1843
+ "pliki_sha256": {
1844
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
1845
+ },
1846
+ "acc": 0.312287,
1847
+ "acc_ci95": [
1848
+ 0.285836,
1849
+ 0.33959
1850
+ ],
1851
+ "acc_norm": 0.330205,
1852
+ "acc_norm_ci95": [
1853
+ 0.302901,
1854
+ 0.356655
1855
+ ],
1856
+ "split": "test",
1857
+ "correct_primary": 387
1858
+ },
1859
+ "hellaswag": {
1860
+ "n": 10042,
1861
+ "n_pelne": 10042,
1862
+ "dataset_repo": "Rowan/hellaswag",
1863
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
1864
+ "pliki_sha256": {
1865
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
1866
+ },
1867
+ "acc": 0.40709,
1868
+ "acc_ci95": [
1869
+ 0.397431,
1870
+ 0.41665
1871
+ ],
1872
+ "acc_norm": 0.520016,
1873
+ "acc_norm_ci95": [
1874
+ 0.510157,
1875
+ 0.529775
1876
+ ],
1877
+ "split": "validation",
1878
+ "correct_primary": 5222
1879
+ },
1880
+ "piqa": {
1881
+ "n": 1838,
1882
+ "n_pelne": 1838,
1883
+ "dataset_repo": "baber/piqa",
1884
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
1885
+ "pliki_sha256": {
1886
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
1887
+ },
1888
+ "acc": 0.707835,
1889
+ "acc_ci95": [
1890
+ 0.68716,
1891
+ 0.728523
1892
+ ],
1893
+ "acc_norm": 0.706746,
1894
+ "acc_norm_ci95": [
1895
+ 0.686072,
1896
+ 0.727421
1897
+ ],
1898
+ "split": "validation",
1899
+ "correct_primary": 1299
1900
+ },
1901
+ "winogrande": {
1902
+ "n": 1267,
1903
+ "n_pelne": 1267,
1904
+ "dataset_repo": "allenai/winogrande",
1905
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
1906
+ "pliki_sha256": {
1907
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
1908
+ },
1909
+ "acc": 0.571429,
1910
+ "acc_ci95": [
1911
+ 0.544574,
1912
+ 0.598264
1913
+ ],
1914
+ "acc_norm": 0.571429,
1915
+ "acc_norm_ci95": [
1916
+ 0.543804,
1917
+ 0.599053
1918
+ ],
1919
+ "split": "validation",
1920
+ "correct_primary": 724
1921
+ },
1922
+ "boolq": {
1923
+ "n": 3270,
1924
+ "n_pelne": 3270,
1925
+ "dataset_repo": "aps/super_glue",
1926
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
1927
+ "pliki_sha256": {
1928
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
1929
+ },
1930
+ "acc": 0.606728,
1931
+ "acc_ci95": [
1932
+ 0.589908,
1933
+ 0.623547
1934
+ ],
1935
+ "acc_norm": 0.622324,
1936
+ "acc_norm_ci95": [
1937
+ 0.605505,
1938
+ 0.638838
1939
+ ],
1940
+ "split": "validation",
1941
+ "correct_primary": 1984
1942
+ },
1943
+ "openbookqa": {
1944
+ "n": 500,
1945
+ "n_pelne": 500,
1946
+ "dataset_repo": "allenai/openbookqa",
1947
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
1948
+ "pliki_sha256": {
1949
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
1950
+ },
1951
+ "acc": 0.242,
1952
+ "acc_ci95": [
1953
+ 0.204,
1954
+ 0.28
1955
+ ],
1956
+ "acc_norm": 0.362,
1957
+ "acc_norm_ci95": [
1958
+ 0.32,
1959
+ 0.404
1960
+ ],
1961
+ "split": "test",
1962
+ "correct_primary": 181
1963
+ },
1964
+ "belebele_pol": {
1965
+ "n": 900,
1966
+ "n_pelne": 900,
1967
+ "dataset_repo": "facebook/belebele",
1968
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
1969
+ "pliki_sha256": {
1970
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
1971
+ },
1972
+ "acc": 0.244444,
1973
+ "acc_ci95": [
1974
+ 0.216667,
1975
+ 0.273333
1976
+ ],
1977
+ "acc_norm": 0.244444,
1978
+ "acc_norm_ci95": [
1979
+ 0.216667,
1980
+ 0.272222
1981
+ ],
1982
+ "split": "pol_Latn evaluation set",
1983
+ "correct_primary": 220
1984
+ },
1985
+ "hellaswag_pl": {
1986
+ "n": 10042,
1987
+ "n_pelne": 10042,
1988
+ "dataset_repo": "Eurolingua/hellaswagx",
1989
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
1990
+ "pliki_sha256": {
1991
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
1992
+ },
1993
+ "acc": 0.360287,
1994
+ "acc_ci95": [
1995
+ 0.350926,
1996
+ 0.369647
1997
+ ],
1998
+ "acc_norm": 0.457479,
1999
+ "acc_norm_ci95": [
2000
+ 0.447717,
2001
+ 0.467337
2002
+ ],
2003
+ "split": "validation",
2004
+ "correct_primary": 4594
2005
+ },
2006
+ "mmlu_pl": {
2007
+ "n": 14042,
2008
+ "n_pelne": 14042,
2009
+ "dataset_repo": "Eurolingua/mmlux",
2010
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
2011
+ "pliki_sha256": {
2012
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
2013
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
2014
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
2015
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
2016
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
2017
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
2018
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
2019
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
2020
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
2021
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
2022
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
2023
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
2024
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
2025
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
2026
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
2027
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
2028
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
2029
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
2030
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
2031
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
2032
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
2033
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
2034
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
2035
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
2036
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
2037
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
2038
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
2039
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
2040
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
2041
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
2042
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
2043
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
2044
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
2045
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
2046
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
2047
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
2048
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
2049
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
2050
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
2051
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
2052
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
2053
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
2054
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
2055
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
2056
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
2057
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
2058
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
2059
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
2060
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
2061
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
2062
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
2063
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
2064
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
2065
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
2066
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
2067
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
2068
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
2069
+ },
2070
+ "acc": 0.247116,
2071
+ "acc_ci95": [
2072
+ 0.239852,
2073
+ 0.254237
2074
+ ],
2075
+ "acc_norm": 0.247116,
2076
+ "acc_norm_ci95": [
2077
+ 0.239994,
2078
+ 0.254237
2079
+ ],
2080
+ "split": "test; micro-average over subjects",
2081
+ "correct_primary": 3470
2082
+ }
2083
+ }
2084
  }
2085
  },
2086
  "additional_measurements": {
README.md CHANGED
@@ -73,31 +73,31 @@ A replay-only continued-pretraining control is not budgeted for the priority run
73
 
74
  Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
75
 
76
- ## Standard benchmark measurements
77
 
78
  We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol work and evaluation proposals. The team's independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is a separate measurement source. **The table below reports SlayerLab evaluations**, also published in the [1.07 training record](https://track.fabryka.ai/run/4973c068-8dd5-59d8-b6c4-671ca3beb925) and the [1.08 training record](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a); it is not a copy of that independent run.
79
 
80
  All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
81
 
82
- For comparison, the continuation columns are **1.09**, not additional 1.08 measurements: updates 5,000, 10,000, 15,000 and 20,000 are distinct intermediate working checkpoints of the ongoing [1.09 run](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a), not inference-ready weight releases or its eventual budget-limited endpoint. Evaluator SHA, dataset pins, file digests, task protocols and sample counts match across these rows.
83
 
84
- | Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 |
85
- |---|---|---:|---:|---:|---:|---:|---:|---:|
86
- | ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] |
87
- | HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] |
88
- | PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] |
89
- | WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] |
90
- | BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] |
91
- | OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] |
92
- | Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] |
93
- | HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] |
94
- | MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] |
95
 
96
  Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The updated approved aggregate does not supply the random-stream derivation or CI-generator SHA; the previous aggregate documented model-specific streams. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
97
 
98
  [Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
99
 
100
- The approved aggregate SHA-256 is `fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796`. The earlier 1.07 and 1.08 scores, intervals and dataset pins are unchanged. The continuation comparison does not extend or replace the separate GSM8K/SST-2 protocol below.
101
 
102
  **Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
103
 
 
73
 
74
  Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
75
 
76
+ ## Standard benchmark measurements: 1.07, 1.08 and 1.09 checkpoints 5k–30k
77
 
78
  We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol work and evaluation proposals. The team's independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is a separate measurement source. **The table below reports SlayerLab evaluations**, also published in the [1.07 training record](https://track.fabryka.ai/run/4973c068-8dd5-59d8-b6c4-671ca3beb925) and the [1.08 training record](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a); it is not a copy of that independent run.
79
 
80
  All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
81
 
82
+ For comparison, the continuation columns are **1.09**, not additional 1.08 measurements: updates 5,000, 10,000, 15,000, 20,000, 25,000 and 30,000 are distinct intermediate working checkpoints of the ongoing [1.09 run](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a), not inference-ready weight releases or its eventual budget-limited endpoint. Evaluator SHA, dataset pins, file digests, task protocols and sample counts match across these rows.
83
 
84
+ | Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 | 1.09, update 25,000 | 1.09, update 30,000 |
85
+ |---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
86
+ | ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] | 33.79 [31.06, 36.52] | 33.02 [30.29, 35.67] |
87
+ | HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] | 51.17 [50.18, 52.14] | 52.00 [51.02, 52.98] |
88
+ | PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] | 71.55 [69.48, 73.61] | 70.67 [68.61, 72.74] |
89
+ | WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] | 55.88 [53.12, 58.64] | 57.14 [54.46, 59.83] |
90
+ | BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] | 52.20 [50.52, 53.91] | 60.67 [58.99, 62.35] |
91
+ | OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] | 37.20 [32.80, 41.40] | 36.20 [32.00, 40.40] |
92
+ | Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] | 25.33 [22.44, 28.22] | 24.44 [21.67, 27.33] |
93
+ | HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] | 43.94 [42.94, 44.91] | 45.75 [44.77, 46.73] |
94
+ | MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] | 24.26 [23.57, 24.96] | 24.71 [23.99, 25.42] |
95
 
96
  Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The updated approved aggregate does not supply the random-stream derivation or CI-generator SHA; the previous aggregate documented model-specific streams. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
97
 
98
  [Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
99
 
100
+ The approved aggregate SHA-256 is `2d257a95523605b06aa964cb974f65d86ab025875cc1f87dc98b242b90729440`. The earlier 1.07, 1.08 and 1.09 updates 5,000–20,000 scores, intervals and dataset pins are unchanged. The continuation comparison does not extend or replace the separate GSM8K/SST-2 protocol below.
101
 
102
  **Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
103