Slayer1B 1.08: update cards and documentation
Browse files- EVALUATION.json +144 -0
- README.md +13 -0
EVALUATION.json
CHANGED
|
@@ -547,5 +547,149 @@
|
|
| 547 |
}
|
| 548 |
}
|
| 549 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 550 |
}
|
| 551 |
}
|
|
|
|
| 547 |
}
|
| 548 |
}
|
| 549 |
}
|
| 550 |
+
},
|
| 551 |
+
"additional_measurements": {
|
| 552 |
+
"benchteam_gsm8k_sst2_20261111": {
|
| 553 |
+
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 554 |
+
"status": "descriptive; training-lineage contamination scan pending",
|
| 555 |
+
"measured_utc_date": "2026-11-11",
|
| 556 |
+
"source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e",
|
| 557 |
+
"protocol": {
|
| 558 |
+
"source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai",
|
| 559 |
+
"inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568",
|
| 560 |
+
"source_protocol_id": "bench50-v0.3.1-fullB",
|
| 561 |
+
"scope": "two-task subset, full evaluated splits; not a full 50-task result",
|
| 562 |
+
"model_adapter": "slayer",
|
| 563 |
+
"lm_eval_version": "0.4.9.1",
|
| 564 |
+
"dtype": "float32",
|
| 565 |
+
"batch_size": 1,
|
| 566 |
+
"context_limit": 4096,
|
| 567 |
+
"task_settings": {
|
| 568 |
+
"gsm8k": {
|
| 569 |
+
"task_id": "T33",
|
| 570 |
+
"split": "test",
|
| 571 |
+
"few_shot": 5,
|
| 572 |
+
"metric": "exact_match,strict-match"
|
| 573 |
+
},
|
| 574 |
+
"sst2": {
|
| 575 |
+
"task_id": "T21",
|
| 576 |
+
"split": "validation",
|
| 577 |
+
"few_shot": 0,
|
| 578 |
+
"metric": "acc,none"
|
| 579 |
+
}
|
| 580 |
+
},
|
| 581 |
+
"metric_units": "fraction; multiply by 100 for card percentages",
|
| 582 |
+
"ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance",
|
| 583 |
+
"dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports"
|
| 584 |
+
},
|
| 585 |
+
"models": {
|
| 586 |
+
"1.07": {
|
| 587 |
+
"step": 22327,
|
| 588 |
+
"weights_published": true,
|
| 589 |
+
"repo": "SlayerLab/Slayer1B-1.07",
|
| 590 |
+
"revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106",
|
| 591 |
+
"weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
|
| 592 |
+
"source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z",
|
| 593 |
+
"source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b",
|
| 594 |
+
"tasks": {
|
| 595 |
+
"gsm8k": {
|
| 596 |
+
"task_id": "T33",
|
| 597 |
+
"metric": "exact_match,strict-match",
|
| 598 |
+
"value": 0.015163002274450341,
|
| 599 |
+
"unit": "score",
|
| 600 |
+
"ci95": {
|
| 601 |
+
"lower": 0.009097801364670205,
|
| 602 |
+
"upper": 0.021986353297952996,
|
| 603 |
+
"level": 0.95,
|
| 604 |
+
"method": "paired_bootstrap",
|
| 605 |
+
"replicates": 10000,
|
| 606 |
+
"seed": 20261028
|
| 607 |
+
},
|
| 608 |
+
"n_scored": 1319,
|
| 609 |
+
"n_population": 1319,
|
| 610 |
+
"contamination": "unscanned",
|
| 611 |
+
"state": "ok",
|
| 612 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 613 |
+
"results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
|
| 614 |
+
"source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
|
| 615 |
+
},
|
| 616 |
+
"sst2": {
|
| 617 |
+
"task_id": "T21",
|
| 618 |
+
"metric": "acc,none",
|
| 619 |
+
"value": 0.5103211009174312,
|
| 620 |
+
"unit": "score",
|
| 621 |
+
"ci95": {
|
| 622 |
+
"lower": 0.47706422018348627,
|
| 623 |
+
"upper": 0.5435779816513762,
|
| 624 |
+
"level": 0.95,
|
| 625 |
+
"method": "paired_bootstrap",
|
| 626 |
+
"replicates": 10000,
|
| 627 |
+
"seed": 20261028
|
| 628 |
+
},
|
| 629 |
+
"n_scored": 872,
|
| 630 |
+
"n_population": 872,
|
| 631 |
+
"contamination": "unscanned",
|
| 632 |
+
"state": "ok",
|
| 633 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 634 |
+
"results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
|
| 635 |
+
"source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
|
| 636 |
+
}
|
| 637 |
+
}
|
| 638 |
+
},
|
| 639 |
+
"1.08": {
|
| 640 |
+
"step": 1709,
|
| 641 |
+
"weights_published": true,
|
| 642 |
+
"repo": "SlayerLab/Slayer1B-1.08",
|
| 643 |
+
"revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c",
|
| 644 |
+
"weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
|
| 645 |
+
"source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z",
|
| 646 |
+
"source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b",
|
| 647 |
+
"tasks": {
|
| 648 |
+
"gsm8k": {
|
| 649 |
+
"task_id": "T33",
|
| 650 |
+
"metric": "exact_match,strict-match",
|
| 651 |
+
"value": 0.009097801364670205,
|
| 652 |
+
"unit": "score",
|
| 653 |
+
"ci95": {
|
| 654 |
+
"lower": 0.004548900682335102,
|
| 655 |
+
"upper": 0.014404852160727824,
|
| 656 |
+
"level": 0.95,
|
| 657 |
+
"method": "paired_bootstrap",
|
| 658 |
+
"replicates": 10000,
|
| 659 |
+
"seed": 20261028
|
| 660 |
+
},
|
| 661 |
+
"n_scored": 1319,
|
| 662 |
+
"n_population": 1319,
|
| 663 |
+
"contamination": "unscanned",
|
| 664 |
+
"state": "ok",
|
| 665 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 666 |
+
"results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6",
|
| 667 |
+
"source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
|
| 668 |
+
},
|
| 669 |
+
"sst2": {
|
| 670 |
+
"task_id": "T21",
|
| 671 |
+
"metric": "acc,none",
|
| 672 |
+
"value": 0.5045871559633027,
|
| 673 |
+
"unit": "score",
|
| 674 |
+
"ci95": {
|
| 675 |
+
"lower": 0.4713302752293578,
|
| 676 |
+
"upper": 0.536697247706422,
|
| 677 |
+
"level": 0.95,
|
| 678 |
+
"method": "paired_bootstrap",
|
| 679 |
+
"replicates": 10000,
|
| 680 |
+
"seed": 20261028
|
| 681 |
+
},
|
| 682 |
+
"n_scored": 872,
|
| 683 |
+
"n_population": 872,
|
| 684 |
+
"contamination": "unscanned",
|
| 685 |
+
"state": "ok",
|
| 686 |
+
"runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
|
| 687 |
+
"results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a",
|
| 688 |
+
"source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
|
| 689 |
+
}
|
| 690 |
+
}
|
| 691 |
+
}
|
| 692 |
+
}
|
| 693 |
+
}
|
| 694 |
}
|
| 695 |
}
|
README.md
CHANGED
|
@@ -99,6 +99,19 @@ Intervals use the percentile bootstrap with 10,000 replicates, seed base 2026111
|
|
| 99 |
|
| 100 |
Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
## Licensing and limitations
|
| 103 |
|
| 104 |
Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.
|
|
|
|
| 99 |
|
| 100 |
Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
|
| 101 |
|
| 102 |
+
## Additional GSM8K and SST-2 measurements
|
| 103 |
+
|
| 104 |
+
SlayerLab measured this two-task subset on 11 November 2026 using the [pinned BenchTeam code](https://github.com/slayerlabs/benchmarks-fabryka-ai/tree/8b547007526971bd2fc35dfe56ff2c7d9b802568), lm-eval 0.4.9.1, the `slayer` adapter, float32, batch 1 and context limit 4,096. These are full evaluated splits, not a full 50-task run or a copy of the community's independent report. The evaluated 1.07 and 1.08 weight identities match the completed checkpoints above.
|
| 105 |
+
|
| 106 |
+
| Task | Metric / prompting | n | 1.07, update 22,327 | 1.08, update 1,709 |
|
| 107 |
+
|---|---|---:|---:|---:|
|
| 108 |
+
| GSM8K | strict exact match, 5-shot | 1,319 | 1.52 [0.91, 2.20] | 0.91 [0.45, 1.44] |
|
| 109 |
+
| SST-2 | accuracy, 0-shot | 872 | 51.03 [47.71, 54.36] | 50.46 [47.13, 53.67] |
|
| 110 |
+
|
| 111 |
+
Scores are percentages with reported pointwise 95% bootstrap intervals (10,000 draws, seed 20261028). These intervals describe individual results, not model differences or training-seed variance. This protocol and its intervals are recorded separately under `additional_measurements` in [EVALUATION.json](EVALUATION.json); the preceding nine-task protocol is unchanged.
|
| 112 |
+
|
| 113 |
+
**Descriptive results; training-lineage contamination scan pending.** GSM8K strict-match scores are near zero. SST-2 zero-shot accuracy is calibration-sensitive; these scores alone do not establish a general reasoning or sentiment-understanding gain. No paired-difference or internal calibration analysis is published here.
|
| 114 |
+
|
| 115 |
## Licensing and limitations
|
| 116 |
|
| 117 |
Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.
|