Text Generation
PyTorch
Polish
English
polish
causal-lm
pretraining
continued-pretraining
training-in-progress
Eval Results (legacy)
Maggio33 commited on
Commit
3910478
·
verified ·
1 Parent(s): 3652dcd

Slayer1B 1.09: update cards and documentation

Browse files
Files changed (2) hide show
  1. EVALUATION.json +197 -0
  2. README.md +13 -0
EVALUATION.json CHANGED
@@ -801,5 +801,202 @@
801
  }
802
  }
803
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
804
  }
805
  }
 
801
  }
802
  }
803
  }
804
+ },
805
+ "additional_measurements": {
806
+ "benchteam_gsm8k_sst2_20261111": {
807
+ "measurement_owner": "SlayerLab / Fabryka AI",
808
+ "status": "descriptive; training-lineage contamination scan pending",
809
+ "measured_utc_date": "2026-11-11",
810
+ "source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e",
811
+ "protocol": {
812
+ "source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai",
813
+ "inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568",
814
+ "source_protocol_id": "bench50-v0.3.1-fullB",
815
+ "scope": "two-task subset, full evaluated splits; not a full 50-task result",
816
+ "model_adapter": "slayer",
817
+ "lm_eval_version": "0.4.9.1",
818
+ "dtype": "float32",
819
+ "batch_size": 1,
820
+ "context_limit": 4096,
821
+ "task_settings": {
822
+ "gsm8k": {
823
+ "task_id": "T33",
824
+ "split": "test",
825
+ "few_shot": 5,
826
+ "metric": "exact_match,strict-match"
827
+ },
828
+ "sst2": {
829
+ "task_id": "T21",
830
+ "split": "validation",
831
+ "few_shot": 0,
832
+ "metric": "acc,none"
833
+ }
834
+ },
835
+ "metric_units": "fraction; multiply by 100 for card percentages",
836
+ "ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance",
837
+ "dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports"
838
+ },
839
+ "models": {
840
+ "1.07": {
841
+ "step": 22327,
842
+ "weights_published": true,
843
+ "repo": "SlayerLab/Slayer1B-1.07",
844
+ "revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106",
845
+ "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
846
+ "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z",
847
+ "source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b",
848
+ "tasks": {
849
+ "gsm8k": {
850
+ "task_id": "T33",
851
+ "metric": "exact_match,strict-match",
852
+ "value": 0.015163002274450341,
853
+ "unit": "score",
854
+ "ci95": {
855
+ "lower": 0.009097801364670205,
856
+ "upper": 0.021986353297952996,
857
+ "level": 0.95,
858
+ "method": "paired_bootstrap",
859
+ "replicates": 10000,
860
+ "seed": 20261028
861
+ },
862
+ "n_scored": 1319,
863
+ "n_population": 1319,
864
+ "contamination": "unscanned",
865
+ "state": "ok",
866
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
867
+ "results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
868
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
869
+ },
870
+ "sst2": {
871
+ "task_id": "T21",
872
+ "metric": "acc,none",
873
+ "value": 0.5103211009174312,
874
+ "unit": "score",
875
+ "ci95": {
876
+ "lower": 0.47706422018348627,
877
+ "upper": 0.5435779816513762,
878
+ "level": 0.95,
879
+ "method": "paired_bootstrap",
880
+ "replicates": 10000,
881
+ "seed": 20261028
882
+ },
883
+ "n_scored": 872,
884
+ "n_population": 872,
885
+ "contamination": "unscanned",
886
+ "state": "ok",
887
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
888
+ "results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
889
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
890
+ }
891
+ }
892
+ },
893
+ "1.08": {
894
+ "step": 1709,
895
+ "weights_published": true,
896
+ "repo": "SlayerLab/Slayer1B-1.08",
897
+ "revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c",
898
+ "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
899
+ "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z",
900
+ "source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b",
901
+ "tasks": {
902
+ "gsm8k": {
903
+ "task_id": "T33",
904
+ "metric": "exact_match,strict-match",
905
+ "value": 0.009097801364670205,
906
+ "unit": "score",
907
+ "ci95": {
908
+ "lower": 0.004548900682335102,
909
+ "upper": 0.014404852160727824,
910
+ "level": 0.95,
911
+ "method": "paired_bootstrap",
912
+ "replicates": 10000,
913
+ "seed": 20261028
914
+ },
915
+ "n_scored": 1319,
916
+ "n_population": 1319,
917
+ "contamination": "unscanned",
918
+ "state": "ok",
919
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
920
+ "results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6",
921
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
922
+ },
923
+ "sst2": {
924
+ "task_id": "T21",
925
+ "metric": "acc,none",
926
+ "value": 0.5045871559633027,
927
+ "unit": "score",
928
+ "ci95": {
929
+ "lower": 0.4713302752293578,
930
+ "upper": 0.536697247706422,
931
+ "level": 0.95,
932
+ "method": "paired_bootstrap",
933
+ "replicates": 10000,
934
+ "seed": 20261028
935
+ },
936
+ "n_scored": 872,
937
+ "n_population": 872,
938
+ "contamination": "unscanned",
939
+ "state": "ok",
940
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
941
+ "results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a",
942
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
943
+ }
944
+ }
945
+ },
946
+ "1.09-k10000": {
947
+ "step": 10000,
948
+ "weights_published": false,
949
+ "repo": "Slayer1B-1.09-k10000-lokalny",
950
+ "revision": null,
951
+ "weights_sha256": "12939d84ef0d06b350773fb30837390306d899ded315d1721b52c7615a87317a",
952
+ "source_run_id": "niezalezny-Slayer1B-1.09-k10000-lokalny-lokalny-20261111T025334Z",
953
+ "source_aggregate_sha256": "042b2639bc325aeb7fb60787bc8205d1384557e37dbb7b01ba5489db41ebdfe8",
954
+ "tasks": {
955
+ "gsm8k": {
956
+ "task_id": "T33",
957
+ "metric": "exact_match,strict-match",
958
+ "value": 0.009855951478392721,
959
+ "unit": "score",
960
+ "ci95": {
961
+ "lower": 0.004548900682335102,
962
+ "upper": 0.015163002274450341,
963
+ "level": 0.95,
964
+ "method": "paired_bootstrap",
965
+ "replicates": 10000,
966
+ "seed": 20261028
967
+ },
968
+ "n_scored": 1319,
969
+ "n_population": 1319,
970
+ "contamination": "unscanned",
971
+ "state": "ok",
972
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
973
+ "results_sha256": "5cf7805fa9c7d3d65d6af44920b6659287c621294efe4d7cde62f2055e65562e",
974
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
975
+ },
976
+ "sst2": {
977
+ "task_id": "T21",
978
+ "metric": "acc,none",
979
+ "value": 0.5825688073394495,
980
+ "unit": "score",
981
+ "ci95": {
982
+ "lower": 0.5493119266055045,
983
+ "upper": 0.6135321100917431,
984
+ "level": 0.95,
985
+ "method": "paired_bootstrap",
986
+ "replicates": 10000,
987
+ "seed": 20261028
988
+ },
989
+ "n_scored": 872,
990
+ "n_population": 872,
991
+ "contamination": "unscanned",
992
+ "state": "ok",
993
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
994
+ "results_sha256": "d47706c62088fe772f1c26d2ed13f52dfecc8d832bbdecd14dd24c89086e9659",
995
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
996
+ }
997
+ }
998
+ }
999
+ }
1000
+ }
1001
  }
1002
  }
README.md CHANGED
@@ -126,6 +126,19 @@ The [machine-readable aggregate record](EVALUATION.json) supplies exact values,
126
 
127
  There is no replay-only control for this continuation, and initialization, exposure and schedule differ between checkpoints. These measurements do not isolate a causal effect of the new datasets or quantify training-seed variance. Earlier internal diagnostics are separate historical records, not a criterion for naming or publishing this model.
128
 
 
 
 
 
 
 
 
 
 
 
 
 
 
129
  ## Cost, licensing and limitations
130
 
131
  The total campaign ceiling is **USD 425**, including GPU allocations, preparation, interruptions, storage and evaluation tracked by the resource meter. On 10 November at 18:57 UTC, the meter reported **USD 141.85 spent**, leaving approximately **USD 283**; this is counter attribution, not a settled invoice or a permanently current balance. Hardware selection is based on effective cost per processed position and verified availability. The final update count is reviewed at stage boundaries and must be selected before its WSD decay boundary. No higher spending limit or guaranteed completion date is implied.
 
126
 
127
  There is no replay-only control for this continuation, and initialization, exposure and schedule differ between checkpoints. These measurements do not isolate a causal effect of the new datasets or quantify training-seed variance. Earlier internal diagnostics are separate historical records, not a criterion for naming or publishing this model.
128
 
129
+ ## Additional GSM8K and SST-2 measurements — update 10,000
130
+
131
+ SlayerLab measured this two-task subset on 11 November 2026 using the [pinned BenchTeam code](https://github.com/slayerlabs/benchmarks-fabryka-ai/tree/8b547007526971bd2fc35dfe56ff2c7d9b802568), lm-eval 0.4.9.1, the `slayer` adapter, float32, batch 1 and context limit 4,096. These are full evaluated splits, not a full 50-task run or a copy of the community's independent report. **Only this two-task section uses 1.09 update 10,000; the nine-task table above remains update 5,000.** Update 10,000 is an internal working checkpoint, not a published or inference-ready 1.09 weight release.
132
+
133
+ | Task | Metric / prompting | n | 1.07 | 1.08 | 1.09, update 10,000 |
134
+ |---|---|---:|---:|---:|---:|
135
+ | GSM8K | strict exact match, 5-shot | 1,319 | 1.52 [0.91, 2.20] | 0.91 [0.45, 1.44] | 0.99 [0.45, 1.52] |
136
+ | SST-2 | accuracy, 0-shot | 872 | 51.03 [47.71, 54.36] | 50.46 [47.13, 53.67] | 58.26 [54.93, 61.35] |
137
+
138
+ Scores are percentages with reported pointwise 95% bootstrap intervals (10,000 draws, seed 20261028). These intervals describe individual results, not model differences or training-seed variance. This protocol and its intervals are recorded separately under `additional_measurements` in [EVALUATION.json](EVALUATION.json); the preceding nine-task protocol is unchanged.
139
+
140
+ **Descriptive results; training-lineage contamination scan pending.** GSM8K strict-match scores are near zero. SST-2 zero-shot accuracy is calibration-sensitive; these scores alone do not establish a general reasoning or sentiment-understanding gain. No paired-difference or internal calibration analysis is published here.
141
+
142
  ## Cost, licensing and limitations
143
 
144
  The total campaign ceiling is **USD 425**, including GPU allocations, preparation, interruptions, storage and evaluation tracked by the resource meter. On 10 November at 18:57 UTC, the meter reported **USD 141.85 spent**, leaving approximately **USD 283**; this is counter attribution, not a settled invoice or a permanently current balance. Hardware selection is based on effective cost per processed position and verified availability. The final update count is reviewed at stage boundaries and must be selected before its WSD decay boundary. No higher spending limit or guaranteed completion date is implied.