Maggio33 commited on
Commit
de5d284
·
verified ·
1 Parent(s): bb637c3

Slayer1B 1.08: update cards and documentation

Browse files
Files changed (2) hide show
  1. EVALUATION.json +144 -0
  2. README.md +13 -0
EVALUATION.json CHANGED
@@ -547,5 +547,149 @@
547
  }
548
  }
549
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
550
  }
551
  }
 
547
  }
548
  }
549
  }
550
+ },
551
+ "additional_measurements": {
552
+ "benchteam_gsm8k_sst2_20261111": {
553
+ "measurement_owner": "SlayerLab / Fabryka AI",
554
+ "status": "descriptive; training-lineage contamination scan pending",
555
+ "measured_utc_date": "2026-11-11",
556
+ "source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e",
557
+ "protocol": {
558
+ "source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai",
559
+ "inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568",
560
+ "source_protocol_id": "bench50-v0.3.1-fullB",
561
+ "scope": "two-task subset, full evaluated splits; not a full 50-task result",
562
+ "model_adapter": "slayer",
563
+ "lm_eval_version": "0.4.9.1",
564
+ "dtype": "float32",
565
+ "batch_size": 1,
566
+ "context_limit": 4096,
567
+ "task_settings": {
568
+ "gsm8k": {
569
+ "task_id": "T33",
570
+ "split": "test",
571
+ "few_shot": 5,
572
+ "metric": "exact_match,strict-match"
573
+ },
574
+ "sst2": {
575
+ "task_id": "T21",
576
+ "split": "validation",
577
+ "few_shot": 0,
578
+ "metric": "acc,none"
579
+ }
580
+ },
581
+ "metric_units": "fraction; multiply by 100 for card percentages",
582
+ "ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance",
583
+ "dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports"
584
+ },
585
+ "models": {
586
+ "1.07": {
587
+ "step": 22327,
588
+ "weights_published": true,
589
+ "repo": "SlayerLab/Slayer1B-1.07",
590
+ "revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106",
591
+ "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
592
+ "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z",
593
+ "source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b",
594
+ "tasks": {
595
+ "gsm8k": {
596
+ "task_id": "T33",
597
+ "metric": "exact_match,strict-match",
598
+ "value": 0.015163002274450341,
599
+ "unit": "score",
600
+ "ci95": {
601
+ "lower": 0.009097801364670205,
602
+ "upper": 0.021986353297952996,
603
+ "level": 0.95,
604
+ "method": "paired_bootstrap",
605
+ "replicates": 10000,
606
+ "seed": 20261028
607
+ },
608
+ "n_scored": 1319,
609
+ "n_population": 1319,
610
+ "contamination": "unscanned",
611
+ "state": "ok",
612
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
613
+ "results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
614
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
615
+ },
616
+ "sst2": {
617
+ "task_id": "T21",
618
+ "metric": "acc,none",
619
+ "value": 0.5103211009174312,
620
+ "unit": "score",
621
+ "ci95": {
622
+ "lower": 0.47706422018348627,
623
+ "upper": 0.5435779816513762,
624
+ "level": 0.95,
625
+ "method": "paired_bootstrap",
626
+ "replicates": 10000,
627
+ "seed": 20261028
628
+ },
629
+ "n_scored": 872,
630
+ "n_population": 872,
631
+ "contamination": "unscanned",
632
+ "state": "ok",
633
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
634
+ "results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
635
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
636
+ }
637
+ }
638
+ },
639
+ "1.08": {
640
+ "step": 1709,
641
+ "weights_published": true,
642
+ "repo": "SlayerLab/Slayer1B-1.08",
643
+ "revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c",
644
+ "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
645
+ "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z",
646
+ "source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b",
647
+ "tasks": {
648
+ "gsm8k": {
649
+ "task_id": "T33",
650
+ "metric": "exact_match,strict-match",
651
+ "value": 0.009097801364670205,
652
+ "unit": "score",
653
+ "ci95": {
654
+ "lower": 0.004548900682335102,
655
+ "upper": 0.014404852160727824,
656
+ "level": 0.95,
657
+ "method": "paired_bootstrap",
658
+ "replicates": 10000,
659
+ "seed": 20261028
660
+ },
661
+ "n_scored": 1319,
662
+ "n_population": 1319,
663
+ "contamination": "unscanned",
664
+ "state": "ok",
665
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
666
+ "results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6",
667
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
668
+ },
669
+ "sst2": {
670
+ "task_id": "T21",
671
+ "metric": "acc,none",
672
+ "value": 0.5045871559633027,
673
+ "unit": "score",
674
+ "ci95": {
675
+ "lower": 0.4713302752293578,
676
+ "upper": 0.536697247706422,
677
+ "level": 0.95,
678
+ "method": "paired_bootstrap",
679
+ "replicates": 10000,
680
+ "seed": 20261028
681
+ },
682
+ "n_scored": 872,
683
+ "n_population": 872,
684
+ "contamination": "unscanned",
685
+ "state": "ok",
686
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
687
+ "results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a",
688
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
689
+ }
690
+ }
691
+ }
692
+ }
693
+ }
694
  }
695
  }
README.md CHANGED
@@ -99,6 +99,19 @@ Intervals use the percentile bootstrap with 10,000 replicates, seed base 2026111
99
 
100
  Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
101
 
 
 
 
 
 
 
 
 
 
 
 
 
 
102
  ## Licensing and limitations
103
 
104
  Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.
 
99
 
100
  Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
101
 
102
+ ## Additional GSM8K and SST-2 measurements
103
+
104
+ SlayerLab measured this two-task subset on 11 November 2026 using the [pinned BenchTeam code](https://github.com/slayerlabs/benchmarks-fabryka-ai/tree/8b547007526971bd2fc35dfe56ff2c7d9b802568), lm-eval 0.4.9.1, the `slayer` adapter, float32, batch 1 and context limit 4,096. These are full evaluated splits, not a full 50-task run or a copy of the community's independent report. The evaluated 1.07 and 1.08 weight identities match the completed checkpoints above.
105
+
106
+ | Task | Metric / prompting | n | 1.07, update 22,327 | 1.08, update 1,709 |
107
+ |---|---|---:|---:|---:|
108
+ | GSM8K | strict exact match, 5-shot | 1,319 | 1.52 [0.91, 2.20] | 0.91 [0.45, 1.44] |
109
+ | SST-2 | accuracy, 0-shot | 872 | 51.03 [47.71, 54.36] | 50.46 [47.13, 53.67] |
110
+
111
+ Scores are percentages with reported pointwise 95% bootstrap intervals (10,000 draws, seed 20261028). These intervals describe individual results, not model differences or training-seed variance. This protocol and its intervals are recorded separately under `additional_measurements` in [EVALUATION.json](EVALUATION.json); the preceding nine-task protocol is unchanged.
112
+
113
+ **Descriptive results; training-lineage contamination scan pending.** GSM8K strict-match scores are near zero. SST-2 zero-shot accuracy is calibration-sensitive; these scores alone do not establish a general reasoning or sentiment-understanding gain. No paired-difference or internal calibration analysis is published here.
114
+
115
  ## Licensing and limitations
116
 
117
  Weights are released under **Apache-2.0**, as 1.07; source corpora keep their own obligations, with attributions in the [data card](DATA_CARD.md). FinePDFs compilation terms and its TDM rationale do not settle rights to every PDF. Book, translation and technical-source notices must be preserved where required.