Slayer1B 1.08: update cards and documentation
Browse files- EVALUATION.json +509 -1
- README.md +14 -14
EVALUATION.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
"status": "descriptive; training-lineage contamination scan pending",
|
| 5 |
-
"source_aggregate_sha256": "
|
| 6 |
"source_generator_sha256": null,
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
|
@@ -1573,6 +1573,514 @@
|
|
| 1573 |
"correct_primary": 3351
|
| 1574 |
}
|
| 1575 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1576 |
}
|
| 1577 |
},
|
| 1578 |
"additional_measurements": {
|
|
|
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
"status": "descriptive; training-lineage contamination scan pending",
|
| 5 |
+
"source_aggregate_sha256": "2d257a95523605b06aa964cb974f65d86ab025875cc1f87dc98b242b90729440",
|
| 6 |
"source_generator_sha256": null,
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
|
|
|
| 1573 |
"correct_primary": 3351
|
| 1574 |
}
|
| 1575 |
}
|
| 1576 |
+
},
|
| 1577 |
+
"1.09-k25000": {
|
| 1578 |
+
"step": 25000,
|
| 1579 |
+
"weights_published": false,
|
| 1580 |
+
"weights_sha256": "46dfedda0d855f36f86fb1a0c0661809c654422c76bd1b3b2259e081f115b310",
|
| 1581 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 1582 |
+
"source_results_sha256": "7b1ac4f7710b11b64542eb193a2820144c3ba6482901159b20356d2928e6f9c5",
|
| 1583 |
+
"tasks": {
|
| 1584 |
+
"arc_challenge": {
|
| 1585 |
+
"n": 1172,
|
| 1586 |
+
"n_pelne": 1172,
|
| 1587 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 1588 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 1589 |
+
"pliki_sha256": {
|
| 1590 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 1591 |
+
},
|
| 1592 |
+
"acc": 0.313993,
|
| 1593 |
+
"acc_ci95": [
|
| 1594 |
+
0.288396,
|
| 1595 |
+
0.340444
|
| 1596 |
+
],
|
| 1597 |
+
"acc_norm": 0.337884,
|
| 1598 |
+
"acc_norm_ci95": [
|
| 1599 |
+
0.31058,
|
| 1600 |
+
0.365188
|
| 1601 |
+
],
|
| 1602 |
+
"split": "test",
|
| 1603 |
+
"correct_primary": 396
|
| 1604 |
+
},
|
| 1605 |
+
"hellaswag": {
|
| 1606 |
+
"n": 10042,
|
| 1607 |
+
"n_pelne": 10042,
|
| 1608 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 1609 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 1610 |
+
"pliki_sha256": {
|
| 1611 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 1612 |
+
},
|
| 1613 |
+
"acc": 0.397132,
|
| 1614 |
+
"acc_ci95": [
|
| 1615 |
+
0.387373,
|
| 1616 |
+
0.406592
|
| 1617 |
+
],
|
| 1618 |
+
"acc_norm": 0.511651,
|
| 1619 |
+
"acc_norm_ci95": [
|
| 1620 |
+
0.501792,
|
| 1621 |
+
0.52141
|
| 1622 |
+
],
|
| 1623 |
+
"split": "validation",
|
| 1624 |
+
"correct_primary": 5138
|
| 1625 |
+
},
|
| 1626 |
+
"piqa": {
|
| 1627 |
+
"n": 1838,
|
| 1628 |
+
"n_pelne": 1838,
|
| 1629 |
+
"dataset_repo": "baber/piqa",
|
| 1630 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 1631 |
+
"pliki_sha256": {
|
| 1632 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 1633 |
+
},
|
| 1634 |
+
"acc": 0.706202,
|
| 1635 |
+
"acc_ci95": [
|
| 1636 |
+
0.685528,
|
| 1637 |
+
0.726877
|
| 1638 |
+
],
|
| 1639 |
+
"acc_norm": 0.715452,
|
| 1640 |
+
"acc_norm_ci95": [
|
| 1641 |
+
0.694763,
|
| 1642 |
+
0.736126
|
| 1643 |
+
],
|
| 1644 |
+
"split": "validation",
|
| 1645 |
+
"correct_primary": 1315
|
| 1646 |
+
},
|
| 1647 |
+
"winogrande": {
|
| 1648 |
+
"n": 1267,
|
| 1649 |
+
"n_pelne": 1267,
|
| 1650 |
+
"dataset_repo": "allenai/winogrande",
|
| 1651 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 1652 |
+
"pliki_sha256": {
|
| 1653 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 1654 |
+
},
|
| 1655 |
+
"acc": 0.5588,
|
| 1656 |
+
"acc_ci95": [
|
| 1657 |
+
0.531176,
|
| 1658 |
+
0.586425
|
| 1659 |
+
],
|
| 1660 |
+
"acc_norm": 0.5588,
|
| 1661 |
+
"acc_norm_ci95": [
|
| 1662 |
+
0.531965,
|
| 1663 |
+
0.586425
|
| 1664 |
+
],
|
| 1665 |
+
"split": "validation",
|
| 1666 |
+
"correct_primary": 708
|
| 1667 |
+
},
|
| 1668 |
+
"boolq": {
|
| 1669 |
+
"n": 3270,
|
| 1670 |
+
"n_pelne": 3270,
|
| 1671 |
+
"dataset_repo": "aps/super_glue",
|
| 1672 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 1673 |
+
"pliki_sha256": {
|
| 1674 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 1675 |
+
},
|
| 1676 |
+
"acc": 0.522018,
|
| 1677 |
+
"acc_ci95": [
|
| 1678 |
+
0.505191,
|
| 1679 |
+
0.539144
|
| 1680 |
+
],
|
| 1681 |
+
"acc_norm": 0.614679,
|
| 1682 |
+
"acc_norm_ci95": [
|
| 1683 |
+
0.598165,
|
| 1684 |
+
0.631498
|
| 1685 |
+
],
|
| 1686 |
+
"split": "validation",
|
| 1687 |
+
"correct_primary": 1707
|
| 1688 |
+
},
|
| 1689 |
+
"openbookqa": {
|
| 1690 |
+
"n": 500,
|
| 1691 |
+
"n_pelne": 500,
|
| 1692 |
+
"dataset_repo": "allenai/openbookqa",
|
| 1693 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 1694 |
+
"pliki_sha256": {
|
| 1695 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 1696 |
+
},
|
| 1697 |
+
"acc": 0.26,
|
| 1698 |
+
"acc_ci95": [
|
| 1699 |
+
0.22,
|
| 1700 |
+
0.3
|
| 1701 |
+
],
|
| 1702 |
+
"acc_norm": 0.372,
|
| 1703 |
+
"acc_norm_ci95": [
|
| 1704 |
+
0.328,
|
| 1705 |
+
0.414
|
| 1706 |
+
],
|
| 1707 |
+
"split": "test",
|
| 1708 |
+
"correct_primary": 186
|
| 1709 |
+
},
|
| 1710 |
+
"belebele_pol": {
|
| 1711 |
+
"n": 900,
|
| 1712 |
+
"n_pelne": 900,
|
| 1713 |
+
"dataset_repo": "facebook/belebele",
|
| 1714 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 1715 |
+
"pliki_sha256": {
|
| 1716 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 1717 |
+
},
|
| 1718 |
+
"acc": 0.253333,
|
| 1719 |
+
"acc_ci95": [
|
| 1720 |
+
0.224444,
|
| 1721 |
+
0.282222
|
| 1722 |
+
],
|
| 1723 |
+
"acc_norm": 0.253333,
|
| 1724 |
+
"acc_norm_ci95": [
|
| 1725 |
+
0.224444,
|
| 1726 |
+
0.282222
|
| 1727 |
+
],
|
| 1728 |
+
"split": "pol_Latn evaluation set",
|
| 1729 |
+
"correct_primary": 228
|
| 1730 |
+
},
|
| 1731 |
+
"hellaswag_pl": {
|
| 1732 |
+
"n": 10042,
|
| 1733 |
+
"n_pelne": 10042,
|
| 1734 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 1735 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 1736 |
+
"pliki_sha256": {
|
| 1737 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 1738 |
+
},
|
| 1739 |
+
"acc": 0.342561,
|
| 1740 |
+
"acc_ci95": [
|
| 1741 |
+
0.333499,
|
| 1742 |
+
0.351524
|
| 1743 |
+
],
|
| 1744 |
+
"acc_norm": 0.439355,
|
| 1745 |
+
"acc_norm_ci95": [
|
| 1746 |
+
0.429397,
|
| 1747 |
+
0.449114
|
| 1748 |
+
],
|
| 1749 |
+
"split": "validation",
|
| 1750 |
+
"correct_primary": 4412
|
| 1751 |
+
},
|
| 1752 |
+
"mmlu_pl": {
|
| 1753 |
+
"n": 14042,
|
| 1754 |
+
"n_pelne": 14042,
|
| 1755 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 1756 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 1757 |
+
"pliki_sha256": {
|
| 1758 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 1759 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 1760 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 1761 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 1762 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 1763 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 1764 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 1765 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 1766 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 1767 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 1768 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 1769 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 1770 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 1771 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 1772 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 1773 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 1774 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 1775 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 1776 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 1777 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 1778 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 1779 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 1780 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 1781 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 1782 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 1783 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 1784 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 1785 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 1786 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 1787 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 1788 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 1789 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 1790 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 1791 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 1792 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 1793 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 1794 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 1795 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 1796 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 1797 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 1798 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 1799 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 1800 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 1801 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 1802 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 1803 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 1804 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 1805 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 1806 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 1807 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 1808 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 1809 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 1810 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 1811 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 1812 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 1813 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 1814 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 1815 |
+
},
|
| 1816 |
+
"acc": 0.242629,
|
| 1817 |
+
"acc_ci95": [
|
| 1818 |
+
0.235721,
|
| 1819 |
+
0.249608
|
| 1820 |
+
],
|
| 1821 |
+
"acc_norm": 0.242629,
|
| 1822 |
+
"acc_norm_ci95": [
|
| 1823 |
+
0.235579,
|
| 1824 |
+
0.24968
|
| 1825 |
+
],
|
| 1826 |
+
"split": "test; micro-average over subjects",
|
| 1827 |
+
"correct_primary": 3407
|
| 1828 |
+
}
|
| 1829 |
+
}
|
| 1830 |
+
},
|
| 1831 |
+
"1.09-k30000": {
|
| 1832 |
+
"step": 30000,
|
| 1833 |
+
"weights_published": false,
|
| 1834 |
+
"weights_sha256": "8a80bb721f6e98677fedcc4ea850896b3f3e5fe158b9f30915e7558bfa1c9764",
|
| 1835 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 1836 |
+
"source_results_sha256": "724d1fa367dcc57d07d4f6959a1cec40b98ddc6350179fc55248586fa1da6a3e",
|
| 1837 |
+
"tasks": {
|
| 1838 |
+
"arc_challenge": {
|
| 1839 |
+
"n": 1172,
|
| 1840 |
+
"n_pelne": 1172,
|
| 1841 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 1842 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 1843 |
+
"pliki_sha256": {
|
| 1844 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 1845 |
+
},
|
| 1846 |
+
"acc": 0.312287,
|
| 1847 |
+
"acc_ci95": [
|
| 1848 |
+
0.285836,
|
| 1849 |
+
0.33959
|
| 1850 |
+
],
|
| 1851 |
+
"acc_norm": 0.330205,
|
| 1852 |
+
"acc_norm_ci95": [
|
| 1853 |
+
0.302901,
|
| 1854 |
+
0.356655
|
| 1855 |
+
],
|
| 1856 |
+
"split": "test",
|
| 1857 |
+
"correct_primary": 387
|
| 1858 |
+
},
|
| 1859 |
+
"hellaswag": {
|
| 1860 |
+
"n": 10042,
|
| 1861 |
+
"n_pelne": 10042,
|
| 1862 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 1863 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 1864 |
+
"pliki_sha256": {
|
| 1865 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 1866 |
+
},
|
| 1867 |
+
"acc": 0.40709,
|
| 1868 |
+
"acc_ci95": [
|
| 1869 |
+
0.397431,
|
| 1870 |
+
0.41665
|
| 1871 |
+
],
|
| 1872 |
+
"acc_norm": 0.520016,
|
| 1873 |
+
"acc_norm_ci95": [
|
| 1874 |
+
0.510157,
|
| 1875 |
+
0.529775
|
| 1876 |
+
],
|
| 1877 |
+
"split": "validation",
|
| 1878 |
+
"correct_primary": 5222
|
| 1879 |
+
},
|
| 1880 |
+
"piqa": {
|
| 1881 |
+
"n": 1838,
|
| 1882 |
+
"n_pelne": 1838,
|
| 1883 |
+
"dataset_repo": "baber/piqa",
|
| 1884 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 1885 |
+
"pliki_sha256": {
|
| 1886 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 1887 |
+
},
|
| 1888 |
+
"acc": 0.707835,
|
| 1889 |
+
"acc_ci95": [
|
| 1890 |
+
0.68716,
|
| 1891 |
+
0.728523
|
| 1892 |
+
],
|
| 1893 |
+
"acc_norm": 0.706746,
|
| 1894 |
+
"acc_norm_ci95": [
|
| 1895 |
+
0.686072,
|
| 1896 |
+
0.727421
|
| 1897 |
+
],
|
| 1898 |
+
"split": "validation",
|
| 1899 |
+
"correct_primary": 1299
|
| 1900 |
+
},
|
| 1901 |
+
"winogrande": {
|
| 1902 |
+
"n": 1267,
|
| 1903 |
+
"n_pelne": 1267,
|
| 1904 |
+
"dataset_repo": "allenai/winogrande",
|
| 1905 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 1906 |
+
"pliki_sha256": {
|
| 1907 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 1908 |
+
},
|
| 1909 |
+
"acc": 0.571429,
|
| 1910 |
+
"acc_ci95": [
|
| 1911 |
+
0.544574,
|
| 1912 |
+
0.598264
|
| 1913 |
+
],
|
| 1914 |
+
"acc_norm": 0.571429,
|
| 1915 |
+
"acc_norm_ci95": [
|
| 1916 |
+
0.543804,
|
| 1917 |
+
0.599053
|
| 1918 |
+
],
|
| 1919 |
+
"split": "validation",
|
| 1920 |
+
"correct_primary": 724
|
| 1921 |
+
},
|
| 1922 |
+
"boolq": {
|
| 1923 |
+
"n": 3270,
|
| 1924 |
+
"n_pelne": 3270,
|
| 1925 |
+
"dataset_repo": "aps/super_glue",
|
| 1926 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 1927 |
+
"pliki_sha256": {
|
| 1928 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 1929 |
+
},
|
| 1930 |
+
"acc": 0.606728,
|
| 1931 |
+
"acc_ci95": [
|
| 1932 |
+
0.589908,
|
| 1933 |
+
0.623547
|
| 1934 |
+
],
|
| 1935 |
+
"acc_norm": 0.622324,
|
| 1936 |
+
"acc_norm_ci95": [
|
| 1937 |
+
0.605505,
|
| 1938 |
+
0.638838
|
| 1939 |
+
],
|
| 1940 |
+
"split": "validation",
|
| 1941 |
+
"correct_primary": 1984
|
| 1942 |
+
},
|
| 1943 |
+
"openbookqa": {
|
| 1944 |
+
"n": 500,
|
| 1945 |
+
"n_pelne": 500,
|
| 1946 |
+
"dataset_repo": "allenai/openbookqa",
|
| 1947 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 1948 |
+
"pliki_sha256": {
|
| 1949 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 1950 |
+
},
|
| 1951 |
+
"acc": 0.242,
|
| 1952 |
+
"acc_ci95": [
|
| 1953 |
+
0.204,
|
| 1954 |
+
0.28
|
| 1955 |
+
],
|
| 1956 |
+
"acc_norm": 0.362,
|
| 1957 |
+
"acc_norm_ci95": [
|
| 1958 |
+
0.32,
|
| 1959 |
+
0.404
|
| 1960 |
+
],
|
| 1961 |
+
"split": "test",
|
| 1962 |
+
"correct_primary": 181
|
| 1963 |
+
},
|
| 1964 |
+
"belebele_pol": {
|
| 1965 |
+
"n": 900,
|
| 1966 |
+
"n_pelne": 900,
|
| 1967 |
+
"dataset_repo": "facebook/belebele",
|
| 1968 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 1969 |
+
"pliki_sha256": {
|
| 1970 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 1971 |
+
},
|
| 1972 |
+
"acc": 0.244444,
|
| 1973 |
+
"acc_ci95": [
|
| 1974 |
+
0.216667,
|
| 1975 |
+
0.273333
|
| 1976 |
+
],
|
| 1977 |
+
"acc_norm": 0.244444,
|
| 1978 |
+
"acc_norm_ci95": [
|
| 1979 |
+
0.216667,
|
| 1980 |
+
0.272222
|
| 1981 |
+
],
|
| 1982 |
+
"split": "pol_Latn evaluation set",
|
| 1983 |
+
"correct_primary": 220
|
| 1984 |
+
},
|
| 1985 |
+
"hellaswag_pl": {
|
| 1986 |
+
"n": 10042,
|
| 1987 |
+
"n_pelne": 10042,
|
| 1988 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 1989 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 1990 |
+
"pliki_sha256": {
|
| 1991 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 1992 |
+
},
|
| 1993 |
+
"acc": 0.360287,
|
| 1994 |
+
"acc_ci95": [
|
| 1995 |
+
0.350926,
|
| 1996 |
+
0.369647
|
| 1997 |
+
],
|
| 1998 |
+
"acc_norm": 0.457479,
|
| 1999 |
+
"acc_norm_ci95": [
|
| 2000 |
+
0.447717,
|
| 2001 |
+
0.467337
|
| 2002 |
+
],
|
| 2003 |
+
"split": "validation",
|
| 2004 |
+
"correct_primary": 4594
|
| 2005 |
+
},
|
| 2006 |
+
"mmlu_pl": {
|
| 2007 |
+
"n": 14042,
|
| 2008 |
+
"n_pelne": 14042,
|
| 2009 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 2010 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 2011 |
+
"pliki_sha256": {
|
| 2012 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 2013 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 2014 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 2015 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 2016 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 2017 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 2018 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 2019 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 2020 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 2021 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 2022 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 2023 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 2024 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 2025 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 2026 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 2027 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 2028 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 2029 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 2030 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 2031 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 2032 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 2033 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 2034 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 2035 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 2036 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 2037 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 2038 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 2039 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 2040 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 2041 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 2042 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 2043 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 2044 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 2045 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 2046 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 2047 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 2048 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 2049 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 2050 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 2051 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 2052 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 2053 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 2054 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 2055 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 2056 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 2057 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 2058 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 2059 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 2060 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 2061 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 2062 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 2063 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 2064 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 2065 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 2066 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 2067 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 2068 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 2069 |
+
},
|
| 2070 |
+
"acc": 0.247116,
|
| 2071 |
+
"acc_ci95": [
|
| 2072 |
+
0.239852,
|
| 2073 |
+
0.254237
|
| 2074 |
+
],
|
| 2075 |
+
"acc_norm": 0.247116,
|
| 2076 |
+
"acc_norm_ci95": [
|
| 2077 |
+
0.239994,
|
| 2078 |
+
0.254237
|
| 2079 |
+
],
|
| 2080 |
+
"split": "test; micro-average over subjects",
|
| 2081 |
+
"correct_primary": 3470
|
| 2082 |
+
}
|
| 2083 |
+
}
|
| 2084 |
}
|
| 2085 |
},
|
| 2086 |
"additional_measurements": {
|
README.md
CHANGED
|
@@ -73,31 +73,31 @@ A replay-only continued-pretraining control is not budgeted for the priority run
|
|
| 73 |
|
| 74 |
Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
|
| 75 |
|
| 76 |
-
## Standard benchmark measurements
|
| 77 |
|
| 78 |
We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol work and evaluation proposals. The team's independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is a separate measurement source. **The table below reports SlayerLab evaluations**, also published in the [1.07 training record](https://track.fabryka.ai/run/4973c068-8dd5-59d8-b6c4-671ca3beb925) and the [1.08 training record](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a); it is not a copy of that independent run.
|
| 79 |
|
| 80 |
All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
|
| 81 |
|
| 82 |
-
For comparison, the continuation columns are **1.09**, not additional 1.08 measurements: updates 5,000, 10,000, 15,000 and
|
| 83 |
|
| 84 |
-
| Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 |
|
| 85 |
-
|---|---|---:|---:|---:|---:|---:|---:|---:|
|
| 86 |
-
| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] |
|
| 87 |
-
| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] |
|
| 88 |
-
| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] |
|
| 89 |
-
| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] |
|
| 90 |
-
| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] |
|
| 91 |
-
| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] |
|
| 92 |
-
| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] |
|
| 93 |
-
| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] |
|
| 94 |
-
| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] |
|
| 95 |
|
| 96 |
Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The updated approved aggregate does not supply the random-stream derivation or CI-generator SHA; the previous aggregate documented model-specific streams. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
|
| 97 |
|
| 98 |
[Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 99 |
|
| 100 |
-
The approved aggregate SHA-256 is `
|
| 101 |
|
| 102 |
**Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
|
| 103 |
|
|
|
|
| 73 |
|
| 74 |
Evaluation records identify checkpoints, evaluator, split, sample count, prompting, normalization and test-read history.
|
| 75 |
|
| 76 |
+
## Standard benchmark measurements: 1.07, 1.08 and 1.09 checkpoints 5k–30k
|
| 77 |
|
| 78 |
We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol work and evaluation proposals. The team's independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is a separate measurement source. **The table below reports SlayerLab evaluations**, also published in the [1.07 training record](https://track.fabryka.ai/run/4973c068-8dd5-59d8-b6c4-671ca3beb925) and the [1.08 training record](https://track.fabryka.ai/run/77126379-f09d-57b8-a0e6-9d9e4e5f1c7a); it is not a copy of that independent run.
|
| 79 |
|
| 80 |
All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
|
| 81 |
|
| 82 |
+
For comparison, the continuation columns are **1.09**, not additional 1.08 measurements: updates 5,000, 10,000, 15,000, 20,000, 25,000 and 30,000 are distinct intermediate working checkpoints of the ongoing [1.09 run](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a), not inference-ready weight releases or its eventual budget-limited endpoint. Evaluator SHA, dataset pins, file digests, task protocols and sample counts match across these rows.
|
| 83 |
|
| 84 |
+
| Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 | 1.09, update 25,000 | 1.09, update 30,000 |
|
| 85 |
+
|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 86 |
+
| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] | 33.79 [31.06, 36.52] | 33.02 [30.29, 35.67] |
|
| 87 |
+
| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] | 51.17 [50.18, 52.14] | 52.00 [51.02, 52.98] |
|
| 88 |
+
| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] | 71.55 [69.48, 73.61] | 70.67 [68.61, 72.74] |
|
| 89 |
+
| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] | 55.88 [53.12, 58.64] | 57.14 [54.46, 59.83] |
|
| 90 |
+
| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] | 52.20 [50.52, 53.91] | 60.67 [58.99, 62.35] |
|
| 91 |
+
| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] | 37.20 [32.80, 41.40] | 36.20 [32.00, 40.40] |
|
| 92 |
+
| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] | 25.33 [22.44, 28.22] | 24.44 [21.67, 27.33] |
|
| 93 |
+
| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] | 43.94 [42.94, 44.91] | 45.75 [44.77, 46.73] |
|
| 94 |
+
| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] | 24.26 [23.57, 24.96] | 24.71 [23.99, 25.42] |
|
| 95 |
|
| 96 |
Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The updated approved aggregate does not supply the random-stream derivation or CI-generator SHA; the previous aggregate documented model-specific streams. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
|
| 97 |
|
| 98 |
[Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 99 |
|
| 100 |
+
The approved aggregate SHA-256 is `2d257a95523605b06aa964cb974f65d86ab025875cc1f87dc98b242b90729440`. The earlier 1.07, 1.08 and 1.09 updates 5,000–20,000 scores, intervals and dataset pins are unchanged. The continuation comparison does not extend or replace the separate GSM8K/SST-2 protocol below.
|
| 101 |
|
| 102 |
**Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
|
| 103 |
|