Slayer1B 1.08: update cards and documentation
Browse files- EVALUATION.json +1175 -147
- README.md +17 -13
EVALUATION.json
CHANGED
|
@@ -2,8 +2,8 @@
|
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
"status": "descriptive; training-lineage contamination scan pending",
|
| 5 |
-
"source_aggregate_sha256": "
|
| 6 |
-
"source_generator_sha256":
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
| 9 |
"protocol": {
|
|
@@ -27,14 +27,25 @@
|
|
| 27 |
"mmlu_pl": "acc"
|
| 28 |
},
|
| 29 |
"mmlu_aggregation": "micro-average over questions",
|
| 30 |
-
"polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
},
|
| 32 |
"ci": {
|
| 33 |
"level": 0.95,
|
| 34 |
"method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
|
| 35 |
"replicates": 10000,
|
| 36 |
"base_seed": 20261110,
|
| 37 |
-
"seed_scope": "
|
| 38 |
"assumption": "items treated as independent; no subject, template or source clusters",
|
| 39 |
"interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
|
| 40 |
},
|
|
@@ -546,150 +557,1167 @@
|
|
| 546 |
"correct_primary": 3235
|
| 547 |
}
|
| 548 |
}
|
| 549 |
-
}
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
"
|
| 554 |
-
"
|
| 555 |
-
"
|
| 556 |
-
"
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
"dtype": "float32",
|
| 565 |
-
"batch_size": 1,
|
| 566 |
-
"context_limit": 4096,
|
| 567 |
-
"task_settings": {
|
| 568 |
-
"gsm8k": {
|
| 569 |
-
"task_id": "T33",
|
| 570 |
-
"split": "test",
|
| 571 |
-
"few_shot": 5,
|
| 572 |
-
"metric": "exact_match,strict-match"
|
| 573 |
},
|
| 574 |
-
"
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 580 |
},
|
| 581 |
-
"
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
"
|
| 590 |
-
"
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
"
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
"lower": 0.009097801364670205,
|
| 602 |
-
"upper": 0.021986353297952996,
|
| 603 |
-
"level": 0.95,
|
| 604 |
-
"method": "paired_bootstrap",
|
| 605 |
-
"replicates": 10000,
|
| 606 |
-
"seed": 20261028
|
| 607 |
-
},
|
| 608 |
-
"n_scored": 1319,
|
| 609 |
-
"n_population": 1319,
|
| 610 |
-
"contamination": "unscanned",
|
| 611 |
-
"state": "ok",
|
| 612 |
-
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 613 |
-
"results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
|
| 614 |
-
"source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
|
| 615 |
-
},
|
| 616 |
-
"sst2": {
|
| 617 |
-
"task_id": "T21",
|
| 618 |
-
"metric": "acc,none",
|
| 619 |
-
"value": 0.5103211009174312,
|
| 620 |
-
"unit": "score",
|
| 621 |
-
"ci95": {
|
| 622 |
-
"lower": 0.47706422018348627,
|
| 623 |
-
"upper": 0.5435779816513762,
|
| 624 |
-
"level": 0.95,
|
| 625 |
-
"method": "paired_bootstrap",
|
| 626 |
-
"replicates": 10000,
|
| 627 |
-
"seed": 20261028
|
| 628 |
-
},
|
| 629 |
-
"n_scored": 872,
|
| 630 |
-
"n_population": 872,
|
| 631 |
-
"contamination": "unscanned",
|
| 632 |
-
"state": "ok",
|
| 633 |
-
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 634 |
-
"results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
|
| 635 |
-
"source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
|
| 636 |
-
}
|
| 637 |
-
}
|
| 638 |
},
|
| 639 |
-
"
|
| 640 |
-
"
|
| 641 |
-
"
|
| 642 |
-
"
|
| 643 |
-
"
|
| 644 |
-
"
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
"
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 695 |
}
|
|
|
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
"status": "descriptive; training-lineage contamination scan pending",
|
| 5 |
+
"source_aggregate_sha256": "fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796",
|
| 6 |
+
"source_generator_sha256": null,
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
| 9 |
"protocol": {
|
|
|
|
| 27 |
"mmlu_pl": "acc"
|
| 28 |
},
|
| 29 |
"mmlu_aggregation": "micro-average over questions",
|
| 30 |
+
"polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b",
|
| 31 |
+
"task_protocols": {
|
| 32 |
+
"arc_challenge": "lm-eval arc_challenge: ctx='Question: {question}\\nAnswer:', kont=' '+choices.text[i], gold=choices.label.index(answerKey); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 33 |
+
"hellaswag": "lm-eval hellaswag: process_docs (preprocess(activity_label+': '+ctx_a+' '+ctx_b.capitalize()), endings preprocess), ctx=query, kont=' '+ending, gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 34 |
+
"piqa": "lm-eval piqa (baber/piqa): ctx='Question: {goal}\\nAnswer:', kont=' '+[sol1,sol2][i], gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 35 |
+
"winogrande": "lm-eval winogrande (partial scoring): ctx=sentence[:idx('_')]+option_i, kont=' '+sentence[idx+1:].strip() (wspólna), gold={'1':0,'2':1}[answer]; 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 36 |
+
"boolq": "lm-eval boolq v2: ctx='{passage}\\nQuestion: {question}?\\nAnswer:', kont=' no'|' yes', gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 37 |
+
"openbookqa": "lm-eval openbookqa: ctx=question_stem, kont=' '+choices.text[i], gold=choices.label.index(answerKey.lstrip()); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 38 |
+
"belebele_pol": "lm-eval belebele_pol_Latn 0-shot: ctx='P: {flores_passage}\\nQ: {question.strip()}\\nA: {mc_answer1}\\nB: ...\\nC: ...\\nD: {mc_answer4}\\nAnswer:', kont=' A'..' D', gold=int(correct_answer_num)-1; 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 39 |
+
"hellaswag_pl": "OpenGPTX ogx_hellaswagx_pl @0018eb9b (= lm-eval hellaswag process_docs na polskich polach); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
|
| 40 |
+
"mmlu_pl": "OpenGPTX ogx_mmlux_pl-* @0018eb9b 0-shot: ctx=description(PL, doklejone BEZ separatora, jak w forku)+'{question.strip()}\\nA. {c0}\\nB. {c1}\\nC. {c2}\\nD. {c3}\\nOdpowiedź:', kont=' A'..' D', gold=answer; acc = mikro po 14042 pozycjach (jak grupa lm-eval mmlu, wagi = rozmiary); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32"
|
| 41 |
+
}
|
| 42 |
},
|
| 43 |
"ci": {
|
| 44 |
"level": 0.95,
|
| 45 |
"method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
|
| 46 |
"replicates": 10000,
|
| 47 |
"base_seed": 20261110,
|
| 48 |
+
"seed_scope": "Updated aggregate records seed 20261110; stream derivation is not supplied. Previous aggregate documented a model-specific stream.",
|
| 49 |
"assumption": "items treated as independent; no subject, template or source clusters",
|
| 50 |
"interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
|
| 51 |
},
|
|
|
|
| 557 |
"correct_primary": 3235
|
| 558 |
}
|
| 559 |
}
|
| 560 |
+
},
|
| 561 |
+
"1.09-k5000": {
|
| 562 |
+
"step": 5000,
|
| 563 |
+
"weights_sha256": "ccd7fc17b834d1de82ac53d0252085cbab567ac96676d0e263cd463a05745631",
|
| 564 |
+
"weights_published": false,
|
| 565 |
+
"source_results_sha256": "ec8d493463d99687f85a06654cef7706b3d37f0c9b9c3e760512f2baac34d632",
|
| 566 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 567 |
+
"tasks": {
|
| 568 |
+
"arc_challenge": {
|
| 569 |
+
"n": 1172,
|
| 570 |
+
"n_pelne": 1172,
|
| 571 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 572 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 573 |
+
"pliki_sha256": {
|
| 574 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 575 |
},
|
| 576 |
+
"acc": 0.288396,
|
| 577 |
+
"acc_ci95": [
|
| 578 |
+
0.262799,
|
| 579 |
+
0.314846
|
| 580 |
+
],
|
| 581 |
+
"acc_norm": 0.293515,
|
| 582 |
+
"acc_norm_ci95": [
|
| 583 |
+
0.267918,
|
| 584 |
+
0.319966
|
| 585 |
+
],
|
| 586 |
+
"split": "test",
|
| 587 |
+
"correct_primary": 344
|
| 588 |
},
|
| 589 |
+
"hellaswag": {
|
| 590 |
+
"n": 10042,
|
| 591 |
+
"n_pelne": 10042,
|
| 592 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 593 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 594 |
+
"pliki_sha256": {
|
| 595 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 596 |
+
},
|
| 597 |
+
"acc": 0.370942,
|
| 598 |
+
"acc_ci95": [
|
| 599 |
+
0.361781,
|
| 600 |
+
0.380303
|
| 601 |
+
],
|
| 602 |
+
"acc_norm": 0.465346,
|
| 603 |
+
"acc_norm_ci95": [
|
| 604 |
+
0.455487,
|
| 605 |
+
0.475204
|
| 606 |
+
],
|
| 607 |
+
"split": "validation",
|
| 608 |
+
"correct_primary": 4673
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 609 |
},
|
| 610 |
+
"piqa": {
|
| 611 |
+
"n": 1838,
|
| 612 |
+
"n_pelne": 1838,
|
| 613 |
+
"dataset_repo": "baber/piqa",
|
| 614 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 615 |
+
"pliki_sha256": {
|
| 616 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 617 |
+
},
|
| 618 |
+
"acc": 0.688248,
|
| 619 |
+
"acc_ci95": [
|
| 620 |
+
0.667029,
|
| 621 |
+
0.708923
|
| 622 |
+
],
|
| 623 |
+
"acc_norm": 0.682807,
|
| 624 |
+
"acc_norm_ci95": [
|
| 625 |
+
0.662119,
|
| 626 |
+
0.704026
|
| 627 |
+
],
|
| 628 |
+
"split": "validation",
|
| 629 |
+
"correct_primary": 1255
|
| 630 |
+
},
|
| 631 |
+
"winogrande": {
|
| 632 |
+
"n": 1267,
|
| 633 |
+
"n_pelne": 1267,
|
| 634 |
+
"dataset_repo": "allenai/winogrande",
|
| 635 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 636 |
+
"pliki_sha256": {
|
| 637 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 638 |
+
},
|
| 639 |
+
"acc": 0.549329,
|
| 640 |
+
"acc_ci95": [
|
| 641 |
+
0.521705,
|
| 642 |
+
0.576953
|
| 643 |
+
],
|
| 644 |
+
"acc_norm": 0.549329,
|
| 645 |
+
"acc_norm_ci95": [
|
| 646 |
+
0.521705,
|
| 647 |
+
0.576953
|
| 648 |
+
],
|
| 649 |
+
"split": "validation",
|
| 650 |
+
"correct_primary": 696
|
| 651 |
+
},
|
| 652 |
+
"boolq": {
|
| 653 |
+
"n": 3270,
|
| 654 |
+
"n_pelne": 3270,
|
| 655 |
+
"dataset_repo": "aps/super_glue",
|
| 656 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 657 |
+
"pliki_sha256": {
|
| 658 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 659 |
+
},
|
| 660 |
+
"acc": 0.618043,
|
| 661 |
+
"acc_ci95": [
|
| 662 |
+
0.601827,
|
| 663 |
+
0.634862
|
| 664 |
+
],
|
| 665 |
+
"acc_norm": 0.623242,
|
| 666 |
+
"acc_norm_ci95": [
|
| 667 |
+
0.606728,
|
| 668 |
+
0.640061
|
| 669 |
+
],
|
| 670 |
+
"split": "validation",
|
| 671 |
+
"correct_primary": 2021
|
| 672 |
+
},
|
| 673 |
+
"openbookqa": {
|
| 674 |
+
"n": 500,
|
| 675 |
+
"n_pelne": 500,
|
| 676 |
+
"dataset_repo": "allenai/openbookqa",
|
| 677 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 678 |
+
"pliki_sha256": {
|
| 679 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 680 |
+
},
|
| 681 |
+
"acc": 0.252,
|
| 682 |
+
"acc_ci95": [
|
| 683 |
+
0.216,
|
| 684 |
+
0.29
|
| 685 |
+
],
|
| 686 |
+
"acc_norm": 0.356,
|
| 687 |
+
"acc_norm_ci95": [
|
| 688 |
+
0.314,
|
| 689 |
+
0.4
|
| 690 |
+
],
|
| 691 |
+
"split": "test",
|
| 692 |
+
"correct_primary": 178
|
| 693 |
+
},
|
| 694 |
+
"belebele_pol": {
|
| 695 |
+
"n": 900,
|
| 696 |
+
"n_pelne": 900,
|
| 697 |
+
"dataset_repo": "facebook/belebele",
|
| 698 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 699 |
+
"pliki_sha256": {
|
| 700 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 701 |
+
},
|
| 702 |
+
"acc": 0.241111,
|
| 703 |
+
"acc_ci95": [
|
| 704 |
+
0.213333,
|
| 705 |
+
0.268889
|
| 706 |
+
],
|
| 707 |
+
"acc_norm": 0.241111,
|
| 708 |
+
"acc_norm_ci95": [
|
| 709 |
+
0.213333,
|
| 710 |
+
0.27
|
| 711 |
+
],
|
| 712 |
+
"split": "pol_Latn evaluation set",
|
| 713 |
+
"correct_primary": 217
|
| 714 |
+
},
|
| 715 |
+
"hellaswag_pl": {
|
| 716 |
+
"n": 10042,
|
| 717 |
+
"n_pelne": 10042,
|
| 718 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 719 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 720 |
+
"pliki_sha256": {
|
| 721 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 722 |
+
},
|
| 723 |
+
"acc": 0.349632,
|
| 724 |
+
"acc_ci95": [
|
| 725 |
+
0.340271,
|
| 726 |
+
0.358992
|
| 727 |
+
],
|
| 728 |
+
"acc_norm": 0.447321,
|
| 729 |
+
"acc_norm_ci95": [
|
| 730 |
+
0.437562,
|
| 731 |
+
0.45708
|
| 732 |
+
],
|
| 733 |
+
"split": "validation",
|
| 734 |
+
"correct_primary": 4492
|
| 735 |
+
},
|
| 736 |
+
"mmlu_pl": {
|
| 737 |
+
"n": 14042,
|
| 738 |
+
"n_pelne": 14042,
|
| 739 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 740 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 741 |
+
"pliki_sha256": {
|
| 742 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 743 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 744 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 745 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 746 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 747 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 748 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 749 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 750 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 751 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 752 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 753 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 754 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 755 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 756 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 757 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 758 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 759 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 760 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 761 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 762 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 763 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 764 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 765 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 766 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 767 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 768 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 769 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 770 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 771 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 772 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 773 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 774 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 775 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 776 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 777 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 778 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 779 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 780 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 781 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 782 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 783 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 784 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 785 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 786 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 787 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 788 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 789 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 790 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 791 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 792 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 793 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 794 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 795 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 796 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 797 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 798 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 799 |
+
},
|
| 800 |
+
"acc": 0.240991,
|
| 801 |
+
"acc_ci95": [
|
| 802 |
+
0.23387,
|
| 803 |
+
0.248184
|
| 804 |
+
],
|
| 805 |
+
"acc_norm": 0.240991,
|
| 806 |
+
"acc_norm_ci95": [
|
| 807 |
+
0.23387,
|
| 808 |
+
0.248042
|
| 809 |
+
],
|
| 810 |
+
"split": "test; micro-average over subjects",
|
| 811 |
+
"correct_primary": 3384
|
| 812 |
+
}
|
| 813 |
+
}
|
| 814 |
+
},
|
| 815 |
+
"1.09-k10000": {
|
| 816 |
+
"step": 10000,
|
| 817 |
+
"weights_sha256": "12939d84ef0d06b350773fb30837390306d899ded315d1721b52c7615a87317a",
|
| 818 |
+
"weights_published": false,
|
| 819 |
+
"source_results_sha256": "3fff987f3b9d74eaba1e121a6a769247fbc12c79043bd5cb5d442c8dafeb9a69",
|
| 820 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 821 |
+
"tasks": {
|
| 822 |
+
"arc_challenge": {
|
| 823 |
+
"n": 1172,
|
| 824 |
+
"n_pelne": 1172,
|
| 825 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 826 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 827 |
+
"pliki_sha256": {
|
| 828 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 829 |
+
},
|
| 830 |
+
"acc": 0.292662,
|
| 831 |
+
"acc_ci95": [
|
| 832 |
+
0.267044,
|
| 833 |
+
0.319113
|
| 834 |
+
],
|
| 835 |
+
"acc_norm": 0.304608,
|
| 836 |
+
"acc_norm_ci95": [
|
| 837 |
+
0.278157,
|
| 838 |
+
0.331058
|
| 839 |
+
],
|
| 840 |
+
"split": "test",
|
| 841 |
+
"correct_primary": 357
|
| 842 |
+
},
|
| 843 |
+
"hellaswag": {
|
| 844 |
+
"n": 10042,
|
| 845 |
+
"n_pelne": 10042,
|
| 846 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 847 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 848 |
+
"pliki_sha256": {
|
| 849 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 850 |
+
},
|
| 851 |
+
"acc": 0.386178,
|
| 852 |
+
"acc_ci95": [
|
| 853 |
+
0.376817,
|
| 854 |
+
0.39574
|
| 855 |
+
],
|
| 856 |
+
"acc_norm": 0.489345,
|
| 857 |
+
"acc_norm_ci95": [
|
| 858 |
+
0.479685,
|
| 859 |
+
0.499104
|
| 860 |
+
],
|
| 861 |
+
"split": "validation",
|
| 862 |
+
"correct_primary": 4914
|
| 863 |
+
},
|
| 864 |
+
"piqa": {
|
| 865 |
+
"n": 1838,
|
| 866 |
+
"n_pelne": 1838,
|
| 867 |
+
"dataset_repo": "baber/piqa",
|
| 868 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 869 |
+
"pliki_sha256": {
|
| 870 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 871 |
+
},
|
| 872 |
+
"acc": 0.696953,
|
| 873 |
+
"acc_ci95": [
|
| 874 |
+
0.675734,
|
| 875 |
+
0.717628
|
| 876 |
+
],
|
| 877 |
+
"acc_norm": 0.68988,
|
| 878 |
+
"acc_norm_ci95": [
|
| 879 |
+
0.669206,
|
| 880 |
+
0.711099
|
| 881 |
+
],
|
| 882 |
+
"split": "validation",
|
| 883 |
+
"correct_primary": 1268
|
| 884 |
+
},
|
| 885 |
+
"winogrande": {
|
| 886 |
+
"n": 1267,
|
| 887 |
+
"n_pelne": 1267,
|
| 888 |
+
"dataset_repo": "allenai/winogrande",
|
| 889 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 890 |
+
"pliki_sha256": {
|
| 891 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 892 |
+
},
|
| 893 |
+
"acc": 0.549329,
|
| 894 |
+
"acc_ci95": [
|
| 895 |
+
0.521705,
|
| 896 |
+
0.576164
|
| 897 |
+
],
|
| 898 |
+
"acc_norm": 0.549329,
|
| 899 |
+
"acc_norm_ci95": [
|
| 900 |
+
0.521705,
|
| 901 |
+
0.576953
|
| 902 |
+
],
|
| 903 |
+
"split": "validation",
|
| 904 |
+
"correct_primary": 696
|
| 905 |
+
},
|
| 906 |
+
"boolq": {
|
| 907 |
+
"n": 3270,
|
| 908 |
+
"n_pelne": 3270,
|
| 909 |
+
"dataset_repo": "aps/super_glue",
|
| 910 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 911 |
+
"pliki_sha256": {
|
| 912 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 913 |
+
},
|
| 914 |
+
"acc": 0.611315,
|
| 915 |
+
"acc_ci95": [
|
| 916 |
+
0.594495,
|
| 917 |
+
0.627829
|
| 918 |
+
],
|
| 919 |
+
"acc_norm": 0.62263,
|
| 920 |
+
"acc_norm_ci95": [
|
| 921 |
+
0.60581,
|
| 922 |
+
0.639144
|
| 923 |
+
],
|
| 924 |
+
"split": "validation",
|
| 925 |
+
"correct_primary": 1999
|
| 926 |
+
},
|
| 927 |
+
"openbookqa": {
|
| 928 |
+
"n": 500,
|
| 929 |
+
"n_pelne": 500,
|
| 930 |
+
"dataset_repo": "allenai/openbookqa",
|
| 931 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 932 |
+
"pliki_sha256": {
|
| 933 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 934 |
+
},
|
| 935 |
+
"acc": 0.242,
|
| 936 |
+
"acc_ci95": [
|
| 937 |
+
0.206,
|
| 938 |
+
0.28
|
| 939 |
+
],
|
| 940 |
+
"acc_norm": 0.344,
|
| 941 |
+
"acc_norm_ci95": [
|
| 942 |
+
0.302,
|
| 943 |
+
0.386
|
| 944 |
+
],
|
| 945 |
+
"split": "test",
|
| 946 |
+
"correct_primary": 172
|
| 947 |
+
},
|
| 948 |
+
"belebele_pol": {
|
| 949 |
+
"n": 900,
|
| 950 |
+
"n_pelne": 900,
|
| 951 |
+
"dataset_repo": "facebook/belebele",
|
| 952 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 953 |
+
"pliki_sha256": {
|
| 954 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 955 |
+
},
|
| 956 |
+
"acc": 0.241111,
|
| 957 |
+
"acc_ci95": [
|
| 958 |
+
0.213333,
|
| 959 |
+
0.27
|
| 960 |
+
],
|
| 961 |
+
"acc_norm": 0.241111,
|
| 962 |
+
"acc_norm_ci95": [
|
| 963 |
+
0.213333,
|
| 964 |
+
0.27
|
| 965 |
+
],
|
| 966 |
+
"split": "pol_Latn evaluation set",
|
| 967 |
+
"correct_primary": 217
|
| 968 |
+
},
|
| 969 |
+
"hellaswag_pl": {
|
| 970 |
+
"n": 10042,
|
| 971 |
+
"n_pelne": 10042,
|
| 972 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 973 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 974 |
+
"pliki_sha256": {
|
| 975 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 976 |
+
},
|
| 977 |
+
"acc": 0.354013,
|
| 978 |
+
"acc_ci95": [
|
| 979 |
+
0.344652,
|
| 980 |
+
0.363274
|
| 981 |
+
],
|
| 982 |
+
"acc_norm": 0.455089,
|
| 983 |
+
"acc_norm_ci95": [
|
| 984 |
+
0.44513,
|
| 985 |
+
0.464848
|
| 986 |
+
],
|
| 987 |
+
"split": "validation",
|
| 988 |
+
"correct_primary": 4570
|
| 989 |
+
},
|
| 990 |
+
"mmlu_pl": {
|
| 991 |
+
"n": 14042,
|
| 992 |
+
"n_pelne": 14042,
|
| 993 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 994 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 995 |
+
"pliki_sha256": {
|
| 996 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 997 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 998 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 999 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 1000 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 1001 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 1002 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 1003 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 1004 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 1005 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 1006 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 1007 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 1008 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 1009 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 1010 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 1011 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 1012 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 1013 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 1014 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 1015 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 1016 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 1017 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 1018 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 1019 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 1020 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 1021 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 1022 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 1023 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 1024 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 1025 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 1026 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 1027 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 1028 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 1029 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 1030 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 1031 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 1032 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 1033 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 1034 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 1035 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 1036 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 1037 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 1038 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 1039 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 1040 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 1041 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 1042 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 1043 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 1044 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 1045 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 1046 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 1047 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 1048 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 1049 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 1050 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 1051 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 1052 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 1053 |
+
},
|
| 1054 |
+
"acc": 0.233229,
|
| 1055 |
+
"acc_ci95": [
|
| 1056 |
+
0.22625,
|
| 1057 |
+
0.240279
|
| 1058 |
+
],
|
| 1059 |
+
"acc_norm": 0.233229,
|
| 1060 |
+
"acc_norm_ci95": [
|
| 1061 |
+
0.22625,
|
| 1062 |
+
0.240422
|
| 1063 |
+
],
|
| 1064 |
+
"split": "test; micro-average over subjects",
|
| 1065 |
+
"correct_primary": 3275
|
| 1066 |
+
}
|
| 1067 |
+
}
|
| 1068 |
+
},
|
| 1069 |
+
"1.09-k15000": {
|
| 1070 |
+
"step": 15000,
|
| 1071 |
+
"weights_sha256": "d5e3927cc5bfe380f6acb9a2de089ef776ae61a57a70d5a719da9f4da3932f9d",
|
| 1072 |
+
"weights_published": false,
|
| 1073 |
+
"source_results_sha256": "1bbb4ce856f45ed4ebacc1911431b749f6df4ed8f5caf292437c9424f03959bd",
|
| 1074 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 1075 |
+
"tasks": {
|
| 1076 |
+
"arc_challenge": {
|
| 1077 |
+
"n": 1172,
|
| 1078 |
+
"n_pelne": 1172,
|
| 1079 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 1080 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 1081 |
+
"pliki_sha256": {
|
| 1082 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 1083 |
+
},
|
| 1084 |
+
"acc": 0.309727,
|
| 1085 |
+
"acc_ci95": [
|
| 1086 |
+
0.283276,
|
| 1087 |
+
0.337031
|
| 1088 |
+
],
|
| 1089 |
+
"acc_norm": 0.325085,
|
| 1090 |
+
"acc_norm_ci95": [
|
| 1091 |
+
0.298635,
|
| 1092 |
+
0.352389
|
| 1093 |
+
],
|
| 1094 |
+
"split": "test",
|
| 1095 |
+
"correct_primary": 381
|
| 1096 |
+
},
|
| 1097 |
+
"hellaswag": {
|
| 1098 |
+
"n": 10042,
|
| 1099 |
+
"n_pelne": 10042,
|
| 1100 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 1101 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 1102 |
+
"pliki_sha256": {
|
| 1103 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 1104 |
+
},
|
| 1105 |
+
"acc": 0.387771,
|
| 1106 |
+
"acc_ci95": [
|
| 1107 |
+
0.378411,
|
| 1108 |
+
0.39753
|
| 1109 |
+
],
|
| 1110 |
+
"acc_norm": 0.49771,
|
| 1111 |
+
"acc_norm_ci95": [
|
| 1112 |
+
0.487951,
|
| 1113 |
+
0.507568
|
| 1114 |
+
],
|
| 1115 |
+
"split": "validation",
|
| 1116 |
+
"correct_primary": 4998
|
| 1117 |
+
},
|
| 1118 |
+
"piqa": {
|
| 1119 |
+
"n": 1838,
|
| 1120 |
+
"n_pelne": 1838,
|
| 1121 |
+
"dataset_repo": "baber/piqa",
|
| 1122 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 1123 |
+
"pliki_sha256": {
|
| 1124 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 1125 |
+
},
|
| 1126 |
+
"acc": 0.695321,
|
| 1127 |
+
"acc_ci95": [
|
| 1128 |
+
0.674646,
|
| 1129 |
+
0.71654
|
| 1130 |
+
],
|
| 1131 |
+
"acc_norm": 0.68988,
|
| 1132 |
+
"acc_norm_ci95": [
|
| 1133 |
+
0.668662,
|
| 1134 |
+
0.710555
|
| 1135 |
+
],
|
| 1136 |
+
"split": "validation",
|
| 1137 |
+
"correct_primary": 1268
|
| 1138 |
+
},
|
| 1139 |
+
"winogrande": {
|
| 1140 |
+
"n": 1267,
|
| 1141 |
+
"n_pelne": 1267,
|
| 1142 |
+
"dataset_repo": "allenai/winogrande",
|
| 1143 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 1144 |
+
"pliki_sha256": {
|
| 1145 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 1146 |
+
},
|
| 1147 |
+
"acc": 0.567482,
|
| 1148 |
+
"acc_ci95": [
|
| 1149 |
+
0.540647,
|
| 1150 |
+
0.594317
|
| 1151 |
+
],
|
| 1152 |
+
"acc_norm": 0.567482,
|
| 1153 |
+
"acc_norm_ci95": [
|
| 1154 |
+
0.539858,
|
| 1155 |
+
0.594317
|
| 1156 |
+
],
|
| 1157 |
+
"split": "validation",
|
| 1158 |
+
"correct_primary": 719
|
| 1159 |
+
},
|
| 1160 |
+
"boolq": {
|
| 1161 |
+
"n": 3270,
|
| 1162 |
+
"n_pelne": 3270,
|
| 1163 |
+
"dataset_repo": "aps/super_glue",
|
| 1164 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 1165 |
+
"pliki_sha256": {
|
| 1166 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 1167 |
+
},
|
| 1168 |
+
"acc": 0.598777,
|
| 1169 |
+
"acc_ci95": [
|
| 1170 |
+
0.581957,
|
| 1171 |
+
0.615291
|
| 1172 |
+
],
|
| 1173 |
+
"acc_norm": 0.619572,
|
| 1174 |
+
"acc_norm_ci95": [
|
| 1175 |
+
0.603058,
|
| 1176 |
+
0.636697
|
| 1177 |
+
],
|
| 1178 |
+
"split": "validation",
|
| 1179 |
+
"correct_primary": 1958
|
| 1180 |
+
},
|
| 1181 |
+
"openbookqa": {
|
| 1182 |
+
"n": 500,
|
| 1183 |
+
"n_pelne": 500,
|
| 1184 |
+
"dataset_repo": "allenai/openbookqa",
|
| 1185 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 1186 |
+
"pliki_sha256": {
|
| 1187 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 1188 |
+
},
|
| 1189 |
+
"acc": 0.252,
|
| 1190 |
+
"acc_ci95": [
|
| 1191 |
+
0.214,
|
| 1192 |
+
0.29
|
| 1193 |
+
],
|
| 1194 |
+
"acc_norm": 0.36,
|
| 1195 |
+
"acc_norm_ci95": [
|
| 1196 |
+
0.318,
|
| 1197 |
+
0.402
|
| 1198 |
+
],
|
| 1199 |
+
"split": "test",
|
| 1200 |
+
"correct_primary": 180
|
| 1201 |
+
},
|
| 1202 |
+
"belebele_pol": {
|
| 1203 |
+
"n": 900,
|
| 1204 |
+
"n_pelne": 900,
|
| 1205 |
+
"dataset_repo": "facebook/belebele",
|
| 1206 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 1207 |
+
"pliki_sha256": {
|
| 1208 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 1209 |
+
},
|
| 1210 |
+
"acc": 0.225556,
|
| 1211 |
+
"acc_ci95": [
|
| 1212 |
+
0.198889,
|
| 1213 |
+
0.253333
|
| 1214 |
+
],
|
| 1215 |
+
"acc_norm": 0.225556,
|
| 1216 |
+
"acc_norm_ci95": [
|
| 1217 |
+
0.197778,
|
| 1218 |
+
0.253333
|
| 1219 |
+
],
|
| 1220 |
+
"split": "pol_Latn evaluation set",
|
| 1221 |
+
"correct_primary": 203
|
| 1222 |
+
},
|
| 1223 |
+
"hellaswag_pl": {
|
| 1224 |
+
"n": 10042,
|
| 1225 |
+
"n_pelne": 10042,
|
| 1226 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 1227 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 1228 |
+
"pliki_sha256": {
|
| 1229 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 1230 |
+
},
|
| 1231 |
+
"acc": 0.34286,
|
| 1232 |
+
"acc_ci95": [
|
| 1233 |
+
0.333599,
|
| 1234 |
+
0.352121
|
| 1235 |
+
],
|
| 1236 |
+
"acc_norm": 0.441844,
|
| 1237 |
+
"acc_norm_ci95": [
|
| 1238 |
+
0.432085,
|
| 1239 |
+
0.451603
|
| 1240 |
+
],
|
| 1241 |
+
"split": "validation",
|
| 1242 |
+
"correct_primary": 4437
|
| 1243 |
+
},
|
| 1244 |
+
"mmlu_pl": {
|
| 1245 |
+
"n": 14042,
|
| 1246 |
+
"n_pelne": 14042,
|
| 1247 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 1248 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 1249 |
+
"pliki_sha256": {
|
| 1250 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 1251 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 1252 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 1253 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 1254 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 1255 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 1256 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 1257 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 1258 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 1259 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 1260 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 1261 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 1262 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 1263 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 1264 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 1265 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 1266 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 1267 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 1268 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 1269 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 1270 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 1271 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 1272 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 1273 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 1274 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 1275 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 1276 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 1277 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 1278 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 1279 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 1280 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 1281 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 1282 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 1283 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 1284 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 1285 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 1286 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 1287 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 1288 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 1289 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 1290 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 1291 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 1292 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 1293 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 1294 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 1295 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 1296 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 1297 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 1298 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 1299 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 1300 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 1301 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 1302 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 1303 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 1304 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 1305 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 1306 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 1307 |
+
},
|
| 1308 |
+
"acc": 0.235009,
|
| 1309 |
+
"acc_ci95": [
|
| 1310 |
+
0.227959,
|
| 1311 |
+
0.24206
|
| 1312 |
+
],
|
| 1313 |
+
"acc_norm": 0.235009,
|
| 1314 |
+
"acc_norm_ci95": [
|
| 1315 |
+
0.22803,
|
| 1316 |
+
0.242273
|
| 1317 |
+
],
|
| 1318 |
+
"split": "test; micro-average over subjects",
|
| 1319 |
+
"correct_primary": 3300
|
| 1320 |
+
}
|
| 1321 |
+
}
|
| 1322 |
+
},
|
| 1323 |
+
"1.09-k20000": {
|
| 1324 |
+
"step": 20000,
|
| 1325 |
+
"weights_sha256": "8bb1927755ae856a5f9f9e63ac2d3f94cd750a8059eeca81be86cf820abc2461",
|
| 1326 |
+
"weights_published": false,
|
| 1327 |
+
"source_results_sha256": "a6225b2af295c846852314f88c0e08f3a71d67d3cd67a53c65bdd87b989a018f",
|
| 1328 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 1329 |
+
"tasks": {
|
| 1330 |
+
"arc_challenge": {
|
| 1331 |
+
"n": 1172,
|
| 1332 |
+
"n_pelne": 1172,
|
| 1333 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 1334 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 1335 |
+
"pliki_sha256": {
|
| 1336 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 1337 |
+
},
|
| 1338 |
+
"acc": 0.306314,
|
| 1339 |
+
"acc_ci95": [
|
| 1340 |
+
0.280717,
|
| 1341 |
+
0.332765
|
| 1342 |
+
],
|
| 1343 |
+
"acc_norm": 0.331058,
|
| 1344 |
+
"acc_norm_ci95": [
|
| 1345 |
+
0.304608,
|
| 1346 |
+
0.358362
|
| 1347 |
+
],
|
| 1348 |
+
"split": "test",
|
| 1349 |
+
"correct_primary": 388
|
| 1350 |
+
},
|
| 1351 |
+
"hellaswag": {
|
| 1352 |
+
"n": 10042,
|
| 1353 |
+
"n_pelne": 10042,
|
| 1354 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 1355 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 1356 |
+
"pliki_sha256": {
|
| 1357 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 1358 |
+
},
|
| 1359 |
+
"acc": 0.397232,
|
| 1360 |
+
"acc_ci95": [
|
| 1361 |
+
0.387473,
|
| 1362 |
+
0.406592
|
| 1363 |
+
],
|
| 1364 |
+
"acc_norm": 0.503784,
|
| 1365 |
+
"acc_norm_ci95": [
|
| 1366 |
+
0.493926,
|
| 1367 |
+
0.513543
|
| 1368 |
+
],
|
| 1369 |
+
"split": "validation",
|
| 1370 |
+
"correct_primary": 5059
|
| 1371 |
+
},
|
| 1372 |
+
"piqa": {
|
| 1373 |
+
"n": 1838,
|
| 1374 |
+
"n_pelne": 1838,
|
| 1375 |
+
"dataset_repo": "baber/piqa",
|
| 1376 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 1377 |
+
"pliki_sha256": {
|
| 1378 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 1379 |
+
},
|
| 1380 |
+
"acc": 0.702938,
|
| 1381 |
+
"acc_ci95": [
|
| 1382 |
+
0.682263,
|
| 1383 |
+
0.723069
|
| 1384 |
+
],
|
| 1385 |
+
"acc_norm": 0.707835,
|
| 1386 |
+
"acc_norm_ci95": [
|
| 1387 |
+
0.68716,
|
| 1388 |
+
0.728509
|
| 1389 |
+
],
|
| 1390 |
+
"split": "validation",
|
| 1391 |
+
"correct_primary": 1301
|
| 1392 |
+
},
|
| 1393 |
+
"winogrande": {
|
| 1394 |
+
"n": 1267,
|
| 1395 |
+
"n_pelne": 1267,
|
| 1396 |
+
"dataset_repo": "allenai/winogrande",
|
| 1397 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 1398 |
+
"pliki_sha256": {
|
| 1399 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 1400 |
+
},
|
| 1401 |
+
"acc": 0.550118,
|
| 1402 |
+
"acc_ci95": [
|
| 1403 |
+
0.522494,
|
| 1404 |
+
0.577743
|
| 1405 |
+
],
|
| 1406 |
+
"acc_norm": 0.550118,
|
| 1407 |
+
"acc_norm_ci95": [
|
| 1408 |
+
0.523283,
|
| 1409 |
+
0.577743
|
| 1410 |
+
],
|
| 1411 |
+
"split": "validation",
|
| 1412 |
+
"correct_primary": 697
|
| 1413 |
+
},
|
| 1414 |
+
"boolq": {
|
| 1415 |
+
"n": 3270,
|
| 1416 |
+
"n_pelne": 3270,
|
| 1417 |
+
"dataset_repo": "aps/super_glue",
|
| 1418 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 1419 |
+
"pliki_sha256": {
|
| 1420 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 1421 |
+
},
|
| 1422 |
+
"acc": 0.514067,
|
| 1423 |
+
"acc_ci95": [
|
| 1424 |
+
0.496942,
|
| 1425 |
+
0.531193
|
| 1426 |
+
],
|
| 1427 |
+
"acc_norm": 0.617737,
|
| 1428 |
+
"acc_norm_ci95": [
|
| 1429 |
+
0.601223,
|
| 1430 |
+
0.634564
|
| 1431 |
+
],
|
| 1432 |
+
"split": "validation",
|
| 1433 |
+
"correct_primary": 1681
|
| 1434 |
+
},
|
| 1435 |
+
"openbookqa": {
|
| 1436 |
+
"n": 500,
|
| 1437 |
+
"n_pelne": 500,
|
| 1438 |
+
"dataset_repo": "allenai/openbookqa",
|
| 1439 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 1440 |
+
"pliki_sha256": {
|
| 1441 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 1442 |
+
},
|
| 1443 |
+
"acc": 0.244,
|
| 1444 |
+
"acc_ci95": [
|
| 1445 |
+
0.208,
|
| 1446 |
+
0.282
|
| 1447 |
+
],
|
| 1448 |
+
"acc_norm": 0.352,
|
| 1449 |
+
"acc_norm_ci95": [
|
| 1450 |
+
0.31,
|
| 1451 |
+
0.394
|
| 1452 |
+
],
|
| 1453 |
+
"split": "test",
|
| 1454 |
+
"correct_primary": 176
|
| 1455 |
+
},
|
| 1456 |
+
"belebele_pol": {
|
| 1457 |
+
"n": 900,
|
| 1458 |
+
"n_pelne": 900,
|
| 1459 |
+
"dataset_repo": "facebook/belebele",
|
| 1460 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 1461 |
+
"pliki_sha256": {
|
| 1462 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 1463 |
+
},
|
| 1464 |
+
"acc": 0.246667,
|
| 1465 |
+
"acc_ci95": [
|
| 1466 |
+
0.218889,
|
| 1467 |
+
0.275556
|
| 1468 |
+
],
|
| 1469 |
+
"acc_norm": 0.246667,
|
| 1470 |
+
"acc_norm_ci95": [
|
| 1471 |
+
0.218889,
|
| 1472 |
+
0.275556
|
| 1473 |
+
],
|
| 1474 |
+
"split": "pol_Latn evaluation set",
|
| 1475 |
+
"correct_primary": 222
|
| 1476 |
+
},
|
| 1477 |
+
"hellaswag_pl": {
|
| 1478 |
+
"n": 10042,
|
| 1479 |
+
"n_pelne": 10042,
|
| 1480 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 1481 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 1482 |
+
"pliki_sha256": {
|
| 1483 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 1484 |
+
},
|
| 1485 |
+
"acc": 0.345549,
|
| 1486 |
+
"acc_ci95": [
|
| 1487 |
+
0.336288,
|
| 1488 |
+
0.355009
|
| 1489 |
+
],
|
| 1490 |
+
"acc_norm": 0.442442,
|
| 1491 |
+
"acc_norm_ci95": [
|
| 1492 |
+
0.432583,
|
| 1493 |
+
0.452201
|
| 1494 |
+
],
|
| 1495 |
+
"split": "validation",
|
| 1496 |
+
"correct_primary": 4443
|
| 1497 |
+
},
|
| 1498 |
+
"mmlu_pl": {
|
| 1499 |
+
"n": 14042,
|
| 1500 |
+
"n_pelne": 14042,
|
| 1501 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 1502 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 1503 |
+
"pliki_sha256": {
|
| 1504 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 1505 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 1506 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 1507 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 1508 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 1509 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 1510 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 1511 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 1512 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 1513 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 1514 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 1515 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 1516 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 1517 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 1518 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 1519 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 1520 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 1521 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 1522 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 1523 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 1524 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 1525 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 1526 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 1527 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 1528 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 1529 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 1530 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 1531 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 1532 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 1533 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 1534 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 1535 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 1536 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 1537 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 1538 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 1539 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 1540 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 1541 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 1542 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 1543 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 1544 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 1545 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 1546 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 1547 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 1548 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 1549 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 1550 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 1551 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 1552 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 1553 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 1554 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 1555 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 1556 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 1557 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 1558 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 1559 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 1560 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 1561 |
+
},
|
| 1562 |
+
"acc": 0.238641,
|
| 1563 |
+
"acc_ci95": [
|
| 1564 |
+
0.231662,
|
| 1565 |
+
0.245763
|
| 1566 |
+
],
|
| 1567 |
+
"acc_norm": 0.238641,
|
| 1568 |
+
"acc_norm_ci95": [
|
| 1569 |
+
0.231591,
|
| 1570 |
+
0.245834
|
| 1571 |
+
],
|
| 1572 |
+
"split": "test; micro-average over subjects",
|
| 1573 |
+
"correct_primary": 3351
|
| 1574 |
+
}
|
| 1575 |
+
}
|
| 1576 |
+
}
|
| 1577 |
+
},
|
| 1578 |
+
"additional_measurements": {
|
| 1579 |
+
"benchteam_gsm8k_sst2_20261111": {
|
| 1580 |
+
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 1581 |
+
"status": "descriptive; training-lineage contamination scan pending",
|
| 1582 |
+
"measured_utc_date": "2026-11-11",
|
| 1583 |
+
"source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e",
|
| 1584 |
+
"protocol": {
|
| 1585 |
+
"source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai",
|
| 1586 |
+
"inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568",
|
| 1587 |
+
"source_protocol_id": "bench50-v0.3.1-fullB",
|
| 1588 |
+
"scope": "two-task subset, full evaluated splits; not a full 50-task result",
|
| 1589 |
+
"model_adapter": "slayer",
|
| 1590 |
+
"lm_eval_version": "0.4.9.1",
|
| 1591 |
+
"dtype": "float32",
|
| 1592 |
+
"batch_size": 1,
|
| 1593 |
+
"context_limit": 4096,
|
| 1594 |
+
"task_settings": {
|
| 1595 |
+
"gsm8k": {
|
| 1596 |
+
"task_id": "T33",
|
| 1597 |
+
"split": "test",
|
| 1598 |
+
"few_shot": 5,
|
| 1599 |
+
"metric": "exact_match,strict-match"
|
| 1600 |
+
},
|
| 1601 |
+
"sst2": {
|
| 1602 |
+
"task_id": "T21",
|
| 1603 |
+
"split": "validation",
|
| 1604 |
+
"few_shot": 0,
|
| 1605 |
+
"metric": "acc,none"
|
| 1606 |
+
}
|
| 1607 |
+
},
|
| 1608 |
+
"metric_units": "fraction; multiply by 100 for card percentages",
|
| 1609 |
+
"ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance",
|
| 1610 |
+
"dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports"
|
| 1611 |
+
},
|
| 1612 |
+
"models": {
|
| 1613 |
+
"1.07": {
|
| 1614 |
+
"step": 22327,
|
| 1615 |
+
"weights_published": true,
|
| 1616 |
+
"repo": "SlayerLab/Slayer1B-1.07",
|
| 1617 |
+
"revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106",
|
| 1618 |
+
"weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
|
| 1619 |
+
"source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z",
|
| 1620 |
+
"source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b",
|
| 1621 |
+
"tasks": {
|
| 1622 |
+
"gsm8k": {
|
| 1623 |
+
"task_id": "T33",
|
| 1624 |
+
"metric": "exact_match,strict-match",
|
| 1625 |
+
"value": 0.015163002274450341,
|
| 1626 |
+
"unit": "score",
|
| 1627 |
+
"ci95": {
|
| 1628 |
+
"lower": 0.009097801364670205,
|
| 1629 |
+
"upper": 0.021986353297952996,
|
| 1630 |
+
"level": 0.95,
|
| 1631 |
+
"method": "paired_bootstrap",
|
| 1632 |
+
"replicates": 10000,
|
| 1633 |
+
"seed": 20261028
|
| 1634 |
+
},
|
| 1635 |
+
"n_scored": 1319,
|
| 1636 |
+
"n_population": 1319,
|
| 1637 |
+
"contamination": "unscanned",
|
| 1638 |
+
"state": "ok",
|
| 1639 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 1640 |
+
"results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
|
| 1641 |
+
"source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
|
| 1642 |
+
},
|
| 1643 |
+
"sst2": {
|
| 1644 |
+
"task_id": "T21",
|
| 1645 |
+
"metric": "acc,none",
|
| 1646 |
+
"value": 0.5103211009174312,
|
| 1647 |
+
"unit": "score",
|
| 1648 |
+
"ci95": {
|
| 1649 |
+
"lower": 0.47706422018348627,
|
| 1650 |
+
"upper": 0.5435779816513762,
|
| 1651 |
+
"level": 0.95,
|
| 1652 |
+
"method": "paired_bootstrap",
|
| 1653 |
+
"replicates": 10000,
|
| 1654 |
+
"seed": 20261028
|
| 1655 |
+
},
|
| 1656 |
+
"n_scored": 872,
|
| 1657 |
+
"n_population": 872,
|
| 1658 |
+
"contamination": "unscanned",
|
| 1659 |
+
"state": "ok",
|
| 1660 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 1661 |
+
"results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
|
| 1662 |
+
"source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
|
| 1663 |
+
}
|
| 1664 |
+
}
|
| 1665 |
+
},
|
| 1666 |
+
"1.08": {
|
| 1667 |
+
"step": 1709,
|
| 1668 |
+
"weights_published": true,
|
| 1669 |
+
"repo": "SlayerLab/Slayer1B-1.08",
|
| 1670 |
+
"revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c",
|
| 1671 |
+
"weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
|
| 1672 |
+
"source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z",
|
| 1673 |
+
"source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b",
|
| 1674 |
+
"tasks": {
|
| 1675 |
+
"gsm8k": {
|
| 1676 |
+
"task_id": "T33",
|
| 1677 |
+
"metric": "exact_match,strict-match",
|
| 1678 |
+
"value": 0.009097801364670205,
|
| 1679 |
+
"unit": "score",
|
| 1680 |
+
"ci95": {
|
| 1681 |
+
"lower": 0.004548900682335102,
|
| 1682 |
+
"upper": 0.014404852160727824,
|
| 1683 |
+
"level": 0.95,
|
| 1684 |
+
"method": "paired_bootstrap",
|
| 1685 |
+
"replicates": 10000,
|
| 1686 |
+
"seed": 20261028
|
| 1687 |
+
},
|
| 1688 |
+
"n_scored": 1319,
|
| 1689 |
+
"n_population": 1319,
|
| 1690 |
+
"contamination": "unscanned",
|
| 1691 |
+
"state": "ok",
|
| 1692 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 1693 |
+
"results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6",
|
| 1694 |
+
"source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
|
| 1695 |
+
},
|
| 1696 |
+
"sst2": {
|
| 1697 |
+
"task_id": "T21",
|
| 1698 |
+
"metric": "acc,none",
|
| 1699 |
+
"value": 0.5045871559633027,
|
| 1700 |
+
"unit": "score",
|
| 1701 |
+
"ci95": {
|
| 1702 |
+
"lower": 0.4713302752293578,
|
| 1703 |
+
"upper": 0.536697247706422,
|
| 1704 |
+
"level": 0.95,
|
| 1705 |
+
"method": "paired_bootstrap",
|
| 1706 |
+
"replicates": 10000,
|
| 1707 |
+
"seed": 20261028
|
| 1708 |
+
},
|
| 1709 |
+
"n_scored": 872,
|
| 1710 |
+
"n_population": 872,
|
| 1711 |
+
"contamination": "unscanned",
|
| 1712 |
+
"state": "ok",
|
| 1713 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 1714 |
+
"results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a",
|
| 1715 |
+
"source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
|
| 1716 |
+
}
|
| 1717 |
+
}
|
| 1718 |
+
}
|
| 1719 |
+
}
|
| 1720 |
+
}
|
| 1721 |
+
},
|
| 1722 |
+
"source_generator_note": "The updated approved aggregate does not supply its CI-generator SHA; evaluator SHA is retained per model. Previous aggregate: ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db; previous CI generator: b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34."
|
| 1723 |
}
|
README.md
CHANGED
|
@@ -79,22 +79,26 @@ We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol
|
|
| 79 |
|
| 80 |
All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
|
| 81 |
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
|
| 85 |
-
|
|
| 86 |
-
|
|
| 87 |
-
|
|
| 88 |
-
|
|
| 89 |
-
|
|
| 90 |
-
|
|
| 91 |
-
|
|
| 92 |
-
|
|
| 93 |
-
|
| 94 |
-
|
|
|
|
|
|
|
| 95 |
|
| 96 |
[Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 97 |
|
|
|
|
|
|
|
| 98 |
**Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
|
| 99 |
|
| 100 |
Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
|
|
|
|
| 79 |
|
| 80 |
All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
|
| 81 |
|
| 82 |
+
For comparison, the continuation columns are **1.09**, not additional 1.08 measurements: updates 5,000, 10,000, 15,000 and 20,000 are distinct intermediate working checkpoints of the ongoing [1.09 run](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a), not inference-ready weight releases or its eventual budget-limited endpoint. Evaluator SHA, dataset pins, file digests, task protocols and sample counts match across these rows.
|
| 83 |
+
|
| 84 |
+
| Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 |
|
| 85 |
+
|---|---|---:|---:|---:|---:|---:|---:|---:|
|
| 86 |
+
| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] |
|
| 87 |
+
| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] |
|
| 88 |
+
| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] |
|
| 89 |
+
| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] |
|
| 90 |
+
| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] |
|
| 91 |
+
| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] |
|
| 92 |
+
| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] |
|
| 93 |
+
| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] |
|
| 94 |
+
| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] |
|
| 95 |
+
|
| 96 |
+
Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The updated approved aggregate does not supply the random-stream derivation or CI-generator SHA; the previous aggregate documented model-specific streams. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
|
| 97 |
|
| 98 |
[Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 99 |
|
| 100 |
+
The approved aggregate SHA-256 is `fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796`. The earlier 1.07 and 1.08 scores, intervals and dataset pins are unchanged. The continuation comparison does not extend or replace the separate GSM8K/SST-2 protocol below.
|
| 101 |
+
|
| 102 |
**Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
|
| 103 |
|
| 104 |
Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
|