Maggio33 commited on
Commit
a64b838
·
verified ·
1 Parent(s): de5d284

Slayer1B 1.08: update cards and documentation

Browse files
Files changed (2) hide show
  1. EVALUATION.json +1175 -147
  2. README.md +17 -13
EVALUATION.json CHANGED
@@ -2,8 +2,8 @@
2
  "schema_version": 1,
3
  "measurement_owner": "SlayerLab / Fabryka AI",
4
  "status": "descriptive; training-lineage contamination scan pending",
5
- "source_aggregate_sha256": "ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db",
6
- "source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
7
  "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
8
  "community_report_is_separate_measurement": true,
9
  "protocol": {
@@ -27,14 +27,25 @@
27
  "mmlu_pl": "acc"
28
  },
29
  "mmlu_aggregation": "micro-average over questions",
30
- "polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b"
 
 
 
 
 
 
 
 
 
 
 
31
  },
32
  "ci": {
33
  "level": 0.95,
34
  "method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
35
  "replicates": 10000,
36
  "base_seed": 20261110,
37
- "seed_scope": "base seed plus model label, as source generator",
38
  "assumption": "items treated as independent; no subject, template or source clusters",
39
  "interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
40
  },
@@ -546,150 +557,1167 @@
546
  "correct_primary": 3235
547
  }
548
  }
549
- }
550
- },
551
- "additional_measurements": {
552
- "benchteam_gsm8k_sst2_20261111": {
553
- "measurement_owner": "SlayerLab / Fabryka AI",
554
- "status": "descriptive; training-lineage contamination scan pending",
555
- "measured_utc_date": "2026-11-11",
556
- "source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e",
557
- "protocol": {
558
- "source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai",
559
- "inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568",
560
- "source_protocol_id": "bench50-v0.3.1-fullB",
561
- "scope": "two-task subset, full evaluated splits; not a full 50-task result",
562
- "model_adapter": "slayer",
563
- "lm_eval_version": "0.4.9.1",
564
- "dtype": "float32",
565
- "batch_size": 1,
566
- "context_limit": 4096,
567
- "task_settings": {
568
- "gsm8k": {
569
- "task_id": "T33",
570
- "split": "test",
571
- "few_shot": 5,
572
- "metric": "exact_match,strict-match"
573
  },
574
- "sst2": {
575
- "task_id": "T21",
576
- "split": "validation",
577
- "few_shot": 0,
578
- "metric": "acc,none"
579
- }
 
 
 
 
 
 
580
  },
581
- "metric_units": "fraction; multiply by 100 for card percentages",
582
- "ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance",
583
- "dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports"
584
- },
585
- "models": {
586
- "1.07": {
587
- "step": 22327,
588
- "weights_published": true,
589
- "repo": "SlayerLab/Slayer1B-1.07",
590
- "revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106",
591
- "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
592
- "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z",
593
- "source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b",
594
- "tasks": {
595
- "gsm8k": {
596
- "task_id": "T33",
597
- "metric": "exact_match,strict-match",
598
- "value": 0.015163002274450341,
599
- "unit": "score",
600
- "ci95": {
601
- "lower": 0.009097801364670205,
602
- "upper": 0.021986353297952996,
603
- "level": 0.95,
604
- "method": "paired_bootstrap",
605
- "replicates": 10000,
606
- "seed": 20261028
607
- },
608
- "n_scored": 1319,
609
- "n_population": 1319,
610
- "contamination": "unscanned",
611
- "state": "ok",
612
- "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
613
- "results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
614
- "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
615
- },
616
- "sst2": {
617
- "task_id": "T21",
618
- "metric": "acc,none",
619
- "value": 0.5103211009174312,
620
- "unit": "score",
621
- "ci95": {
622
- "lower": 0.47706422018348627,
623
- "upper": 0.5435779816513762,
624
- "level": 0.95,
625
- "method": "paired_bootstrap",
626
- "replicates": 10000,
627
- "seed": 20261028
628
- },
629
- "n_scored": 872,
630
- "n_population": 872,
631
- "contamination": "unscanned",
632
- "state": "ok",
633
- "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
634
- "results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
635
- "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
636
- }
637
- }
638
  },
639
- "1.08": {
640
- "step": 1709,
641
- "weights_published": true,
642
- "repo": "SlayerLab/Slayer1B-1.08",
643
- "revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c",
644
- "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
645
- "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z",
646
- "source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b",
647
- "tasks": {
648
- "gsm8k": {
649
- "task_id": "T33",
650
- "metric": "exact_match,strict-match",
651
- "value": 0.009097801364670205,
652
- "unit": "score",
653
- "ci95": {
654
- "lower": 0.004548900682335102,
655
- "upper": 0.014404852160727824,
656
- "level": 0.95,
657
- "method": "paired_bootstrap",
658
- "replicates": 10000,
659
- "seed": 20261028
660
- },
661
- "n_scored": 1319,
662
- "n_population": 1319,
663
- "contamination": "unscanned",
664
- "state": "ok",
665
- "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
666
- "results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6",
667
- "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
668
- },
669
- "sst2": {
670
- "task_id": "T21",
671
- "metric": "acc,none",
672
- "value": 0.5045871559633027,
673
- "unit": "score",
674
- "ci95": {
675
- "lower": 0.4713302752293578,
676
- "upper": 0.536697247706422,
677
- "level": 0.95,
678
- "method": "paired_bootstrap",
679
- "replicates": 10000,
680
- "seed": 20261028
681
- },
682
- "n_scored": 872,
683
- "n_population": 872,
684
- "contamination": "unscanned",
685
- "state": "ok",
686
- "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
687
- "results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a",
688
- "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
689
- }
690
- }
691
- }
692
- }
693
- }
694
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
695
  }
 
2
  "schema_version": 1,
3
  "measurement_owner": "SlayerLab / Fabryka AI",
4
  "status": "descriptive; training-lineage contamination scan pending",
5
+ "source_aggregate_sha256": "fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796",
6
+ "source_generator_sha256": null,
7
  "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
8
  "community_report_is_separate_measurement": true,
9
  "protocol": {
 
27
  "mmlu_pl": "acc"
28
  },
29
  "mmlu_aggregation": "micro-average over questions",
30
+ "polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b",
31
+ "task_protocols": {
32
+ "arc_challenge": "lm-eval arc_challenge: ctx='Question: {question}\\nAnswer:', kont=' '+choices.text[i], gold=choices.label.index(answerKey); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
33
+ "hellaswag": "lm-eval hellaswag: process_docs (preprocess(activity_label+': '+ctx_a+' '+ctx_b.capitalize()), endings preprocess), ctx=query, kont=' '+ending, gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
34
+ "piqa": "lm-eval piqa (baber/piqa): ctx='Question: {goal}\\nAnswer:', kont=' '+[sol1,sol2][i], gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
35
+ "winogrande": "lm-eval winogrande (partial scoring): ctx=sentence[:idx('_')]+option_i, kont=' '+sentence[idx+1:].strip() (wspólna), gold={'1':0,'2':1}[answer]; 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
36
+ "boolq": "lm-eval boolq v2: ctx='{passage}\\nQuestion: {question}?\\nAnswer:', kont=' no'|' yes', gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
37
+ "openbookqa": "lm-eval openbookqa: ctx=question_stem, kont=' '+choices.text[i], gold=choices.label.index(answerKey.lstrip()); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
38
+ "belebele_pol": "lm-eval belebele_pol_Latn 0-shot: ctx='P: {flores_passage}\\nQ: {question.strip()}\\nA: {mc_answer1}\\nB: ...\\nC: ...\\nD: {mc_answer4}\\nAnswer:', kont=' A'..' D', gold=int(correct_answer_num)-1; 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
39
+ "hellaswag_pl": "OpenGPTX ogx_hellaswagx_pl @0018eb9b (= lm-eval hellaswag process_docs na polskich polach); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32",
40
+ "mmlu_pl": "OpenGPTX ogx_mmlux_pl-* @0018eb9b 0-shot: ctx=description(PL, doklejone BEZ separatora, jak w forku)+'{question.strip()}\\nA. {c0}\\nB. {c1}\\nC. {c2}\\nD. {c3}\\nOdpowiedź:', kont=' A'..' D', gold=answer; acc = mikro po 14042 pozycjach (jak grupa lm-eval mmlu, wagi = rozmiary); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32"
41
+ }
42
  },
43
  "ci": {
44
  "level": 0.95,
45
  "method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
46
  "replicates": 10000,
47
  "base_seed": 20261110,
48
+ "seed_scope": "Updated aggregate records seed 20261110; stream derivation is not supplied. Previous aggregate documented a model-specific stream.",
49
  "assumption": "items treated as independent; no subject, template or source clusters",
50
  "interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
51
  },
 
557
  "correct_primary": 3235
558
  }
559
  }
560
+ },
561
+ "1.09-k5000": {
562
+ "step": 5000,
563
+ "weights_sha256": "ccd7fc17b834d1de82ac53d0252085cbab567ac96676d0e263cd463a05745631",
564
+ "weights_published": false,
565
+ "source_results_sha256": "ec8d493463d99687f85a06654cef7706b3d37f0c9b9c3e760512f2baac34d632",
566
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
567
+ "tasks": {
568
+ "arc_challenge": {
569
+ "n": 1172,
570
+ "n_pelne": 1172,
571
+ "dataset_repo": "allenai/ai2_arc",
572
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
573
+ "pliki_sha256": {
574
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
 
 
 
 
 
 
 
 
 
575
  },
576
+ "acc": 0.288396,
577
+ "acc_ci95": [
578
+ 0.262799,
579
+ 0.314846
580
+ ],
581
+ "acc_norm": 0.293515,
582
+ "acc_norm_ci95": [
583
+ 0.267918,
584
+ 0.319966
585
+ ],
586
+ "split": "test",
587
+ "correct_primary": 344
588
  },
589
+ "hellaswag": {
590
+ "n": 10042,
591
+ "n_pelne": 10042,
592
+ "dataset_repo": "Rowan/hellaswag",
593
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
594
+ "pliki_sha256": {
595
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
596
+ },
597
+ "acc": 0.370942,
598
+ "acc_ci95": [
599
+ 0.361781,
600
+ 0.380303
601
+ ],
602
+ "acc_norm": 0.465346,
603
+ "acc_norm_ci95": [
604
+ 0.455487,
605
+ 0.475204
606
+ ],
607
+ "split": "validation",
608
+ "correct_primary": 4673
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
609
  },
610
+ "piqa": {
611
+ "n": 1838,
612
+ "n_pelne": 1838,
613
+ "dataset_repo": "baber/piqa",
614
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
615
+ "pliki_sha256": {
616
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
617
+ },
618
+ "acc": 0.688248,
619
+ "acc_ci95": [
620
+ 0.667029,
621
+ 0.708923
622
+ ],
623
+ "acc_norm": 0.682807,
624
+ "acc_norm_ci95": [
625
+ 0.662119,
626
+ 0.704026
627
+ ],
628
+ "split": "validation",
629
+ "correct_primary": 1255
630
+ },
631
+ "winogrande": {
632
+ "n": 1267,
633
+ "n_pelne": 1267,
634
+ "dataset_repo": "allenai/winogrande",
635
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
636
+ "pliki_sha256": {
637
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
638
+ },
639
+ "acc": 0.549329,
640
+ "acc_ci95": [
641
+ 0.521705,
642
+ 0.576953
643
+ ],
644
+ "acc_norm": 0.549329,
645
+ "acc_norm_ci95": [
646
+ 0.521705,
647
+ 0.576953
648
+ ],
649
+ "split": "validation",
650
+ "correct_primary": 696
651
+ },
652
+ "boolq": {
653
+ "n": 3270,
654
+ "n_pelne": 3270,
655
+ "dataset_repo": "aps/super_glue",
656
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
657
+ "pliki_sha256": {
658
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
659
+ },
660
+ "acc": 0.618043,
661
+ "acc_ci95": [
662
+ 0.601827,
663
+ 0.634862
664
+ ],
665
+ "acc_norm": 0.623242,
666
+ "acc_norm_ci95": [
667
+ 0.606728,
668
+ 0.640061
669
+ ],
670
+ "split": "validation",
671
+ "correct_primary": 2021
672
+ },
673
+ "openbookqa": {
674
+ "n": 500,
675
+ "n_pelne": 500,
676
+ "dataset_repo": "allenai/openbookqa",
677
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
678
+ "pliki_sha256": {
679
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
680
+ },
681
+ "acc": 0.252,
682
+ "acc_ci95": [
683
+ 0.216,
684
+ 0.29
685
+ ],
686
+ "acc_norm": 0.356,
687
+ "acc_norm_ci95": [
688
+ 0.314,
689
+ 0.4
690
+ ],
691
+ "split": "test",
692
+ "correct_primary": 178
693
+ },
694
+ "belebele_pol": {
695
+ "n": 900,
696
+ "n_pelne": 900,
697
+ "dataset_repo": "facebook/belebele",
698
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
699
+ "pliki_sha256": {
700
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
701
+ },
702
+ "acc": 0.241111,
703
+ "acc_ci95": [
704
+ 0.213333,
705
+ 0.268889
706
+ ],
707
+ "acc_norm": 0.241111,
708
+ "acc_norm_ci95": [
709
+ 0.213333,
710
+ 0.27
711
+ ],
712
+ "split": "pol_Latn evaluation set",
713
+ "correct_primary": 217
714
+ },
715
+ "hellaswag_pl": {
716
+ "n": 10042,
717
+ "n_pelne": 10042,
718
+ "dataset_repo": "Eurolingua/hellaswagx",
719
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
720
+ "pliki_sha256": {
721
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
722
+ },
723
+ "acc": 0.349632,
724
+ "acc_ci95": [
725
+ 0.340271,
726
+ 0.358992
727
+ ],
728
+ "acc_norm": 0.447321,
729
+ "acc_norm_ci95": [
730
+ 0.437562,
731
+ 0.45708
732
+ ],
733
+ "split": "validation",
734
+ "correct_primary": 4492
735
+ },
736
+ "mmlu_pl": {
737
+ "n": 14042,
738
+ "n_pelne": 14042,
739
+ "dataset_repo": "Eurolingua/mmlux",
740
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
741
+ "pliki_sha256": {
742
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
743
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
744
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
745
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
746
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
747
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
748
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
749
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
750
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
751
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
752
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
753
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
754
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
755
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
756
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
757
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
758
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
759
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
760
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
761
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
762
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
763
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
764
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
765
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
766
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
767
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
768
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
769
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
770
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
771
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
772
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
773
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
774
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
775
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
776
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
777
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
778
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
779
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
780
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
781
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
782
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
783
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
784
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
785
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
786
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
787
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
788
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
789
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
790
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
791
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
792
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
793
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
794
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
795
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
796
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
797
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
798
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
799
+ },
800
+ "acc": 0.240991,
801
+ "acc_ci95": [
802
+ 0.23387,
803
+ 0.248184
804
+ ],
805
+ "acc_norm": 0.240991,
806
+ "acc_norm_ci95": [
807
+ 0.23387,
808
+ 0.248042
809
+ ],
810
+ "split": "test; micro-average over subjects",
811
+ "correct_primary": 3384
812
+ }
813
+ }
814
+ },
815
+ "1.09-k10000": {
816
+ "step": 10000,
817
+ "weights_sha256": "12939d84ef0d06b350773fb30837390306d899ded315d1721b52c7615a87317a",
818
+ "weights_published": false,
819
+ "source_results_sha256": "3fff987f3b9d74eaba1e121a6a769247fbc12c79043bd5cb5d442c8dafeb9a69",
820
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
821
+ "tasks": {
822
+ "arc_challenge": {
823
+ "n": 1172,
824
+ "n_pelne": 1172,
825
+ "dataset_repo": "allenai/ai2_arc",
826
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
827
+ "pliki_sha256": {
828
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
829
+ },
830
+ "acc": 0.292662,
831
+ "acc_ci95": [
832
+ 0.267044,
833
+ 0.319113
834
+ ],
835
+ "acc_norm": 0.304608,
836
+ "acc_norm_ci95": [
837
+ 0.278157,
838
+ 0.331058
839
+ ],
840
+ "split": "test",
841
+ "correct_primary": 357
842
+ },
843
+ "hellaswag": {
844
+ "n": 10042,
845
+ "n_pelne": 10042,
846
+ "dataset_repo": "Rowan/hellaswag",
847
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
848
+ "pliki_sha256": {
849
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
850
+ },
851
+ "acc": 0.386178,
852
+ "acc_ci95": [
853
+ 0.376817,
854
+ 0.39574
855
+ ],
856
+ "acc_norm": 0.489345,
857
+ "acc_norm_ci95": [
858
+ 0.479685,
859
+ 0.499104
860
+ ],
861
+ "split": "validation",
862
+ "correct_primary": 4914
863
+ },
864
+ "piqa": {
865
+ "n": 1838,
866
+ "n_pelne": 1838,
867
+ "dataset_repo": "baber/piqa",
868
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
869
+ "pliki_sha256": {
870
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
871
+ },
872
+ "acc": 0.696953,
873
+ "acc_ci95": [
874
+ 0.675734,
875
+ 0.717628
876
+ ],
877
+ "acc_norm": 0.68988,
878
+ "acc_norm_ci95": [
879
+ 0.669206,
880
+ 0.711099
881
+ ],
882
+ "split": "validation",
883
+ "correct_primary": 1268
884
+ },
885
+ "winogrande": {
886
+ "n": 1267,
887
+ "n_pelne": 1267,
888
+ "dataset_repo": "allenai/winogrande",
889
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
890
+ "pliki_sha256": {
891
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
892
+ },
893
+ "acc": 0.549329,
894
+ "acc_ci95": [
895
+ 0.521705,
896
+ 0.576164
897
+ ],
898
+ "acc_norm": 0.549329,
899
+ "acc_norm_ci95": [
900
+ 0.521705,
901
+ 0.576953
902
+ ],
903
+ "split": "validation",
904
+ "correct_primary": 696
905
+ },
906
+ "boolq": {
907
+ "n": 3270,
908
+ "n_pelne": 3270,
909
+ "dataset_repo": "aps/super_glue",
910
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
911
+ "pliki_sha256": {
912
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
913
+ },
914
+ "acc": 0.611315,
915
+ "acc_ci95": [
916
+ 0.594495,
917
+ 0.627829
918
+ ],
919
+ "acc_norm": 0.62263,
920
+ "acc_norm_ci95": [
921
+ 0.60581,
922
+ 0.639144
923
+ ],
924
+ "split": "validation",
925
+ "correct_primary": 1999
926
+ },
927
+ "openbookqa": {
928
+ "n": 500,
929
+ "n_pelne": 500,
930
+ "dataset_repo": "allenai/openbookqa",
931
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
932
+ "pliki_sha256": {
933
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
934
+ },
935
+ "acc": 0.242,
936
+ "acc_ci95": [
937
+ 0.206,
938
+ 0.28
939
+ ],
940
+ "acc_norm": 0.344,
941
+ "acc_norm_ci95": [
942
+ 0.302,
943
+ 0.386
944
+ ],
945
+ "split": "test",
946
+ "correct_primary": 172
947
+ },
948
+ "belebele_pol": {
949
+ "n": 900,
950
+ "n_pelne": 900,
951
+ "dataset_repo": "facebook/belebele",
952
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
953
+ "pliki_sha256": {
954
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
955
+ },
956
+ "acc": 0.241111,
957
+ "acc_ci95": [
958
+ 0.213333,
959
+ 0.27
960
+ ],
961
+ "acc_norm": 0.241111,
962
+ "acc_norm_ci95": [
963
+ 0.213333,
964
+ 0.27
965
+ ],
966
+ "split": "pol_Latn evaluation set",
967
+ "correct_primary": 217
968
+ },
969
+ "hellaswag_pl": {
970
+ "n": 10042,
971
+ "n_pelne": 10042,
972
+ "dataset_repo": "Eurolingua/hellaswagx",
973
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
974
+ "pliki_sha256": {
975
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
976
+ },
977
+ "acc": 0.354013,
978
+ "acc_ci95": [
979
+ 0.344652,
980
+ 0.363274
981
+ ],
982
+ "acc_norm": 0.455089,
983
+ "acc_norm_ci95": [
984
+ 0.44513,
985
+ 0.464848
986
+ ],
987
+ "split": "validation",
988
+ "correct_primary": 4570
989
+ },
990
+ "mmlu_pl": {
991
+ "n": 14042,
992
+ "n_pelne": 14042,
993
+ "dataset_repo": "Eurolingua/mmlux",
994
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
995
+ "pliki_sha256": {
996
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
997
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
998
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
999
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
1000
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
1001
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
1002
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
1003
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
1004
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
1005
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
1006
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
1007
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
1008
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
1009
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
1010
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
1011
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
1012
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
1013
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
1014
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
1015
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
1016
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
1017
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
1018
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
1019
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
1020
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
1021
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
1022
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
1023
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
1024
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
1025
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
1026
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
1027
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
1028
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
1029
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
1030
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
1031
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
1032
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
1033
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
1034
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
1035
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
1036
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
1037
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
1038
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
1039
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
1040
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
1041
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
1042
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
1043
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
1044
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
1045
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
1046
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
1047
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
1048
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
1049
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
1050
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
1051
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
1052
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
1053
+ },
1054
+ "acc": 0.233229,
1055
+ "acc_ci95": [
1056
+ 0.22625,
1057
+ 0.240279
1058
+ ],
1059
+ "acc_norm": 0.233229,
1060
+ "acc_norm_ci95": [
1061
+ 0.22625,
1062
+ 0.240422
1063
+ ],
1064
+ "split": "test; micro-average over subjects",
1065
+ "correct_primary": 3275
1066
+ }
1067
+ }
1068
+ },
1069
+ "1.09-k15000": {
1070
+ "step": 15000,
1071
+ "weights_sha256": "d5e3927cc5bfe380f6acb9a2de089ef776ae61a57a70d5a719da9f4da3932f9d",
1072
+ "weights_published": false,
1073
+ "source_results_sha256": "1bbb4ce856f45ed4ebacc1911431b749f6df4ed8f5caf292437c9424f03959bd",
1074
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
1075
+ "tasks": {
1076
+ "arc_challenge": {
1077
+ "n": 1172,
1078
+ "n_pelne": 1172,
1079
+ "dataset_repo": "allenai/ai2_arc",
1080
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
1081
+ "pliki_sha256": {
1082
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
1083
+ },
1084
+ "acc": 0.309727,
1085
+ "acc_ci95": [
1086
+ 0.283276,
1087
+ 0.337031
1088
+ ],
1089
+ "acc_norm": 0.325085,
1090
+ "acc_norm_ci95": [
1091
+ 0.298635,
1092
+ 0.352389
1093
+ ],
1094
+ "split": "test",
1095
+ "correct_primary": 381
1096
+ },
1097
+ "hellaswag": {
1098
+ "n": 10042,
1099
+ "n_pelne": 10042,
1100
+ "dataset_repo": "Rowan/hellaswag",
1101
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
1102
+ "pliki_sha256": {
1103
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
1104
+ },
1105
+ "acc": 0.387771,
1106
+ "acc_ci95": [
1107
+ 0.378411,
1108
+ 0.39753
1109
+ ],
1110
+ "acc_norm": 0.49771,
1111
+ "acc_norm_ci95": [
1112
+ 0.487951,
1113
+ 0.507568
1114
+ ],
1115
+ "split": "validation",
1116
+ "correct_primary": 4998
1117
+ },
1118
+ "piqa": {
1119
+ "n": 1838,
1120
+ "n_pelne": 1838,
1121
+ "dataset_repo": "baber/piqa",
1122
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
1123
+ "pliki_sha256": {
1124
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
1125
+ },
1126
+ "acc": 0.695321,
1127
+ "acc_ci95": [
1128
+ 0.674646,
1129
+ 0.71654
1130
+ ],
1131
+ "acc_norm": 0.68988,
1132
+ "acc_norm_ci95": [
1133
+ 0.668662,
1134
+ 0.710555
1135
+ ],
1136
+ "split": "validation",
1137
+ "correct_primary": 1268
1138
+ },
1139
+ "winogrande": {
1140
+ "n": 1267,
1141
+ "n_pelne": 1267,
1142
+ "dataset_repo": "allenai/winogrande",
1143
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
1144
+ "pliki_sha256": {
1145
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
1146
+ },
1147
+ "acc": 0.567482,
1148
+ "acc_ci95": [
1149
+ 0.540647,
1150
+ 0.594317
1151
+ ],
1152
+ "acc_norm": 0.567482,
1153
+ "acc_norm_ci95": [
1154
+ 0.539858,
1155
+ 0.594317
1156
+ ],
1157
+ "split": "validation",
1158
+ "correct_primary": 719
1159
+ },
1160
+ "boolq": {
1161
+ "n": 3270,
1162
+ "n_pelne": 3270,
1163
+ "dataset_repo": "aps/super_glue",
1164
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
1165
+ "pliki_sha256": {
1166
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
1167
+ },
1168
+ "acc": 0.598777,
1169
+ "acc_ci95": [
1170
+ 0.581957,
1171
+ 0.615291
1172
+ ],
1173
+ "acc_norm": 0.619572,
1174
+ "acc_norm_ci95": [
1175
+ 0.603058,
1176
+ 0.636697
1177
+ ],
1178
+ "split": "validation",
1179
+ "correct_primary": 1958
1180
+ },
1181
+ "openbookqa": {
1182
+ "n": 500,
1183
+ "n_pelne": 500,
1184
+ "dataset_repo": "allenai/openbookqa",
1185
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
1186
+ "pliki_sha256": {
1187
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
1188
+ },
1189
+ "acc": 0.252,
1190
+ "acc_ci95": [
1191
+ 0.214,
1192
+ 0.29
1193
+ ],
1194
+ "acc_norm": 0.36,
1195
+ "acc_norm_ci95": [
1196
+ 0.318,
1197
+ 0.402
1198
+ ],
1199
+ "split": "test",
1200
+ "correct_primary": 180
1201
+ },
1202
+ "belebele_pol": {
1203
+ "n": 900,
1204
+ "n_pelne": 900,
1205
+ "dataset_repo": "facebook/belebele",
1206
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
1207
+ "pliki_sha256": {
1208
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
1209
+ },
1210
+ "acc": 0.225556,
1211
+ "acc_ci95": [
1212
+ 0.198889,
1213
+ 0.253333
1214
+ ],
1215
+ "acc_norm": 0.225556,
1216
+ "acc_norm_ci95": [
1217
+ 0.197778,
1218
+ 0.253333
1219
+ ],
1220
+ "split": "pol_Latn evaluation set",
1221
+ "correct_primary": 203
1222
+ },
1223
+ "hellaswag_pl": {
1224
+ "n": 10042,
1225
+ "n_pelne": 10042,
1226
+ "dataset_repo": "Eurolingua/hellaswagx",
1227
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
1228
+ "pliki_sha256": {
1229
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
1230
+ },
1231
+ "acc": 0.34286,
1232
+ "acc_ci95": [
1233
+ 0.333599,
1234
+ 0.352121
1235
+ ],
1236
+ "acc_norm": 0.441844,
1237
+ "acc_norm_ci95": [
1238
+ 0.432085,
1239
+ 0.451603
1240
+ ],
1241
+ "split": "validation",
1242
+ "correct_primary": 4437
1243
+ },
1244
+ "mmlu_pl": {
1245
+ "n": 14042,
1246
+ "n_pelne": 14042,
1247
+ "dataset_repo": "Eurolingua/mmlux",
1248
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
1249
+ "pliki_sha256": {
1250
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
1251
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
1252
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
1253
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
1254
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
1255
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
1256
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
1257
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
1258
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
1259
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
1260
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
1261
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
1262
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
1263
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
1264
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
1265
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
1266
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
1267
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
1268
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
1269
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
1270
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
1271
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
1272
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
1273
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
1274
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
1275
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
1276
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
1277
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
1278
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
1279
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
1280
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
1281
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
1282
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
1283
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
1284
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
1285
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
1286
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
1287
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
1288
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
1289
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
1290
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
1291
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
1292
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
1293
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
1294
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
1295
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
1296
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
1297
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
1298
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
1299
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
1300
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
1301
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
1302
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
1303
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
1304
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
1305
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
1306
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
1307
+ },
1308
+ "acc": 0.235009,
1309
+ "acc_ci95": [
1310
+ 0.227959,
1311
+ 0.24206
1312
+ ],
1313
+ "acc_norm": 0.235009,
1314
+ "acc_norm_ci95": [
1315
+ 0.22803,
1316
+ 0.242273
1317
+ ],
1318
+ "split": "test; micro-average over subjects",
1319
+ "correct_primary": 3300
1320
+ }
1321
+ }
1322
+ },
1323
+ "1.09-k20000": {
1324
+ "step": 20000,
1325
+ "weights_sha256": "8bb1927755ae856a5f9f9e63ac2d3f94cd750a8059eeca81be86cf820abc2461",
1326
+ "weights_published": false,
1327
+ "source_results_sha256": "a6225b2af295c846852314f88c0e08f3a71d67d3cd67a53c65bdd87b989a018f",
1328
+ "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
1329
+ "tasks": {
1330
+ "arc_challenge": {
1331
+ "n": 1172,
1332
+ "n_pelne": 1172,
1333
+ "dataset_repo": "allenai/ai2_arc",
1334
+ "dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
1335
+ "pliki_sha256": {
1336
+ "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
1337
+ },
1338
+ "acc": 0.306314,
1339
+ "acc_ci95": [
1340
+ 0.280717,
1341
+ 0.332765
1342
+ ],
1343
+ "acc_norm": 0.331058,
1344
+ "acc_norm_ci95": [
1345
+ 0.304608,
1346
+ 0.358362
1347
+ ],
1348
+ "split": "test",
1349
+ "correct_primary": 388
1350
+ },
1351
+ "hellaswag": {
1352
+ "n": 10042,
1353
+ "n_pelne": 10042,
1354
+ "dataset_repo": "Rowan/hellaswag",
1355
+ "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
1356
+ "pliki_sha256": {
1357
+ "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
1358
+ },
1359
+ "acc": 0.397232,
1360
+ "acc_ci95": [
1361
+ 0.387473,
1362
+ 0.406592
1363
+ ],
1364
+ "acc_norm": 0.503784,
1365
+ "acc_norm_ci95": [
1366
+ 0.493926,
1367
+ 0.513543
1368
+ ],
1369
+ "split": "validation",
1370
+ "correct_primary": 5059
1371
+ },
1372
+ "piqa": {
1373
+ "n": 1838,
1374
+ "n_pelne": 1838,
1375
+ "dataset_repo": "baber/piqa",
1376
+ "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
1377
+ "pliki_sha256": {
1378
+ "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
1379
+ },
1380
+ "acc": 0.702938,
1381
+ "acc_ci95": [
1382
+ 0.682263,
1383
+ 0.723069
1384
+ ],
1385
+ "acc_norm": 0.707835,
1386
+ "acc_norm_ci95": [
1387
+ 0.68716,
1388
+ 0.728509
1389
+ ],
1390
+ "split": "validation",
1391
+ "correct_primary": 1301
1392
+ },
1393
+ "winogrande": {
1394
+ "n": 1267,
1395
+ "n_pelne": 1267,
1396
+ "dataset_repo": "allenai/winogrande",
1397
+ "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
1398
+ "pliki_sha256": {
1399
+ "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
1400
+ },
1401
+ "acc": 0.550118,
1402
+ "acc_ci95": [
1403
+ 0.522494,
1404
+ 0.577743
1405
+ ],
1406
+ "acc_norm": 0.550118,
1407
+ "acc_norm_ci95": [
1408
+ 0.523283,
1409
+ 0.577743
1410
+ ],
1411
+ "split": "validation",
1412
+ "correct_primary": 697
1413
+ },
1414
+ "boolq": {
1415
+ "n": 3270,
1416
+ "n_pelne": 3270,
1417
+ "dataset_repo": "aps/super_glue",
1418
+ "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
1419
+ "pliki_sha256": {
1420
+ "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
1421
+ },
1422
+ "acc": 0.514067,
1423
+ "acc_ci95": [
1424
+ 0.496942,
1425
+ 0.531193
1426
+ ],
1427
+ "acc_norm": 0.617737,
1428
+ "acc_norm_ci95": [
1429
+ 0.601223,
1430
+ 0.634564
1431
+ ],
1432
+ "split": "validation",
1433
+ "correct_primary": 1681
1434
+ },
1435
+ "openbookqa": {
1436
+ "n": 500,
1437
+ "n_pelne": 500,
1438
+ "dataset_repo": "allenai/openbookqa",
1439
+ "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
1440
+ "pliki_sha256": {
1441
+ "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
1442
+ },
1443
+ "acc": 0.244,
1444
+ "acc_ci95": [
1445
+ 0.208,
1446
+ 0.282
1447
+ ],
1448
+ "acc_norm": 0.352,
1449
+ "acc_norm_ci95": [
1450
+ 0.31,
1451
+ 0.394
1452
+ ],
1453
+ "split": "test",
1454
+ "correct_primary": 176
1455
+ },
1456
+ "belebele_pol": {
1457
+ "n": 900,
1458
+ "n_pelne": 900,
1459
+ "dataset_repo": "facebook/belebele",
1460
+ "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
1461
+ "pliki_sha256": {
1462
+ "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
1463
+ },
1464
+ "acc": 0.246667,
1465
+ "acc_ci95": [
1466
+ 0.218889,
1467
+ 0.275556
1468
+ ],
1469
+ "acc_norm": 0.246667,
1470
+ "acc_norm_ci95": [
1471
+ 0.218889,
1472
+ 0.275556
1473
+ ],
1474
+ "split": "pol_Latn evaluation set",
1475
+ "correct_primary": 222
1476
+ },
1477
+ "hellaswag_pl": {
1478
+ "n": 10042,
1479
+ "n_pelne": 10042,
1480
+ "dataset_repo": "Eurolingua/hellaswagx",
1481
+ "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
1482
+ "pliki_sha256": {
1483
+ "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
1484
+ },
1485
+ "acc": 0.345549,
1486
+ "acc_ci95": [
1487
+ 0.336288,
1488
+ 0.355009
1489
+ ],
1490
+ "acc_norm": 0.442442,
1491
+ "acc_norm_ci95": [
1492
+ 0.432583,
1493
+ 0.452201
1494
+ ],
1495
+ "split": "validation",
1496
+ "correct_primary": 4443
1497
+ },
1498
+ "mmlu_pl": {
1499
+ "n": 14042,
1500
+ "n_pelne": 14042,
1501
+ "dataset_repo": "Eurolingua/mmlux",
1502
+ "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
1503
+ "pliki_sha256": {
1504
+ "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
1505
+ "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
1506
+ "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
1507
+ "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
1508
+ "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
1509
+ "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
1510
+ "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
1511
+ "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
1512
+ "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
1513
+ "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
1514
+ "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
1515
+ "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
1516
+ "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
1517
+ "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
1518
+ "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
1519
+ "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
1520
+ "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
1521
+ "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
1522
+ "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
1523
+ "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
1524
+ "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
1525
+ "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
1526
+ "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
1527
+ "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
1528
+ "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
1529
+ "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
1530
+ "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
1531
+ "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
1532
+ "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
1533
+ "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
1534
+ "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
1535
+ "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
1536
+ "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
1537
+ "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
1538
+ "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
1539
+ "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
1540
+ "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
1541
+ "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
1542
+ "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
1543
+ "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
1544
+ "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
1545
+ "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
1546
+ "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
1547
+ "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
1548
+ "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
1549
+ "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
1550
+ "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
1551
+ "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
1552
+ "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
1553
+ "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
1554
+ "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
1555
+ "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
1556
+ "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
1557
+ "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
1558
+ "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
1559
+ "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
1560
+ "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
1561
+ },
1562
+ "acc": 0.238641,
1563
+ "acc_ci95": [
1564
+ 0.231662,
1565
+ 0.245763
1566
+ ],
1567
+ "acc_norm": 0.238641,
1568
+ "acc_norm_ci95": [
1569
+ 0.231591,
1570
+ 0.245834
1571
+ ],
1572
+ "split": "test; micro-average over subjects",
1573
+ "correct_primary": 3351
1574
+ }
1575
+ }
1576
+ }
1577
+ },
1578
+ "additional_measurements": {
1579
+ "benchteam_gsm8k_sst2_20261111": {
1580
+ "measurement_owner": "SlayerLab / Fabryka AI",
1581
+ "status": "descriptive; training-lineage contamination scan pending",
1582
+ "measured_utc_date": "2026-11-11",
1583
+ "source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e",
1584
+ "protocol": {
1585
+ "source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai",
1586
+ "inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568",
1587
+ "source_protocol_id": "bench50-v0.3.1-fullB",
1588
+ "scope": "two-task subset, full evaluated splits; not a full 50-task result",
1589
+ "model_adapter": "slayer",
1590
+ "lm_eval_version": "0.4.9.1",
1591
+ "dtype": "float32",
1592
+ "batch_size": 1,
1593
+ "context_limit": 4096,
1594
+ "task_settings": {
1595
+ "gsm8k": {
1596
+ "task_id": "T33",
1597
+ "split": "test",
1598
+ "few_shot": 5,
1599
+ "metric": "exact_match,strict-match"
1600
+ },
1601
+ "sst2": {
1602
+ "task_id": "T21",
1603
+ "split": "validation",
1604
+ "few_shot": 0,
1605
+ "metric": "acc,none"
1606
+ }
1607
+ },
1608
+ "metric_units": "fraction; multiply by 100 for card percentages",
1609
+ "ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance",
1610
+ "dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports"
1611
+ },
1612
+ "models": {
1613
+ "1.07": {
1614
+ "step": 22327,
1615
+ "weights_published": true,
1616
+ "repo": "SlayerLab/Slayer1B-1.07",
1617
+ "revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106",
1618
+ "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5",
1619
+ "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z",
1620
+ "source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b",
1621
+ "tasks": {
1622
+ "gsm8k": {
1623
+ "task_id": "T33",
1624
+ "metric": "exact_match,strict-match",
1625
+ "value": 0.015163002274450341,
1626
+ "unit": "score",
1627
+ "ci95": {
1628
+ "lower": 0.009097801364670205,
1629
+ "upper": 0.021986353297952996,
1630
+ "level": 0.95,
1631
+ "method": "paired_bootstrap",
1632
+ "replicates": 10000,
1633
+ "seed": 20261028
1634
+ },
1635
+ "n_scored": 1319,
1636
+ "n_population": 1319,
1637
+ "contamination": "unscanned",
1638
+ "state": "ok",
1639
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
1640
+ "results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6",
1641
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
1642
+ },
1643
+ "sst2": {
1644
+ "task_id": "T21",
1645
+ "metric": "acc,none",
1646
+ "value": 0.5103211009174312,
1647
+ "unit": "score",
1648
+ "ci95": {
1649
+ "lower": 0.47706422018348627,
1650
+ "upper": 0.5435779816513762,
1651
+ "level": 0.95,
1652
+ "method": "paired_bootstrap",
1653
+ "replicates": 10000,
1654
+ "seed": 20261028
1655
+ },
1656
+ "n_scored": 872,
1657
+ "n_population": 872,
1658
+ "contamination": "unscanned",
1659
+ "state": "ok",
1660
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
1661
+ "results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f",
1662
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
1663
+ }
1664
+ }
1665
+ },
1666
+ "1.08": {
1667
+ "step": 1709,
1668
+ "weights_published": true,
1669
+ "repo": "SlayerLab/Slayer1B-1.08",
1670
+ "revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c",
1671
+ "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4",
1672
+ "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z",
1673
+ "source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b",
1674
+ "tasks": {
1675
+ "gsm8k": {
1676
+ "task_id": "T33",
1677
+ "metric": "exact_match,strict-match",
1678
+ "value": 0.009097801364670205,
1679
+ "unit": "score",
1680
+ "ci95": {
1681
+ "lower": 0.004548900682335102,
1682
+ "upper": 0.014404852160727824,
1683
+ "level": 0.95,
1684
+ "method": "paired_bootstrap",
1685
+ "replicates": 10000,
1686
+ "seed": 20261028
1687
+ },
1688
+ "n_scored": 1319,
1689
+ "n_population": 1319,
1690
+ "contamination": "unscanned",
1691
+ "state": "ok",
1692
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
1693
+ "results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6",
1694
+ "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0"
1695
+ },
1696
+ "sst2": {
1697
+ "task_id": "T21",
1698
+ "metric": "acc,none",
1699
+ "value": 0.5045871559633027,
1700
+ "unit": "score",
1701
+ "ci95": {
1702
+ "lower": 0.4713302752293578,
1703
+ "upper": 0.536697247706422,
1704
+ "level": 0.95,
1705
+ "method": "paired_bootstrap",
1706
+ "replicates": 10000,
1707
+ "seed": 20261028
1708
+ },
1709
+ "n_scored": 872,
1710
+ "n_population": 872,
1711
+ "contamination": "unscanned",
1712
+ "state": "ok",
1713
+ "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e",
1714
+ "results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a",
1715
+ "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7"
1716
+ }
1717
+ }
1718
+ }
1719
+ }
1720
+ }
1721
+ },
1722
+ "source_generator_note": "The updated approved aggregate does not supply its CI-generator SHA; evaluator SHA is retained per model. Previous aggregate: ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db; previous CI generator: b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34."
1723
  }
README.md CHANGED
@@ -79,22 +79,26 @@ We thank **BenchTeam / Fabryka AI** for its benchmark implementations, protocol
79
 
80
  All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
81
 
82
- | Task | Metric | n | 1.07, update 22,327 | 1.08, update 1,709 |
83
- |---|---|---:|---:|---:|
84
- | ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] |
85
- | HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] |
86
- | PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] |
87
- | WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] |
88
- | BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] |
89
- | OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] |
90
- | Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] |
91
- | HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] |
92
- | MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] |
93
-
94
- Intervals use the percentile bootstrap with 10,000 replicates, seed base 20261110 and a model-specific stream. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
 
 
95
 
96
  [Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
97
 
 
 
98
  **Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
99
 
100
  Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.
 
79
 
80
  All nine tasks use their full evaluated split, zero-shot likelihood scoring and float32. Scores are percentages with **pointwise 95% confidence intervals**. `acc` is accuracy from summed continuation likelihood; `acc_norm` is the evaluator's length-normalized accuracy. The metric is named per task rather than combined into a score. WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged over 14,042 questions. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
81
 
82
+ For comparison, the continuation columns are **1.09**, not additional 1.08 measurements: updates 5,000, 10,000, 15,000 and 20,000 are distinct intermediate working checkpoints of the ongoing [1.09 run](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a), not inference-ready weight releases or its eventual budget-limited endpoint. Evaluator SHA, dataset pins, file digests, task protocols and sample counts match across these rows.
83
+
84
+ | Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 |
85
+ |---|---|---:|---:|---:|---:|---:|---:|---:|
86
+ | ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] |
87
+ | HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] |
88
+ | PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] |
89
+ | WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] |
90
+ | BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] |
91
+ | OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] |
92
+ | Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] |
93
+ | HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] |
94
+ | MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] |
95
+
96
+ Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The updated approved aggregate does not supply the random-stream derivation or CI-generator SHA; the previous aggregate documented model-specific streams. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired intervals for model differences or estimates of training-seed variance. They do not support a difference claim based on interval overlap.
97
 
98
  [Machine-readable aggregate record](EVALUATION.json): exact scores, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection. It contains no prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
99
 
100
+ The approved aggregate SHA-256 is `fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796`. The earlier 1.07 and 1.08 scores, intervals and dataset pins are unchanged. The continuation comparison does not extend or replace the separate GSM8K/SST-2 protocol below.
101
+
102
  **Descriptive results; contamination scan of the training lineage against these tasks is pending.** Belebele and MMLU-PL results are near their four-choice 25% random baseline. No overall benchmark average, ranking or model-release threshold is inferred from this table. Results from other evaluators can differ with prompting, BOS handling, normalization or library version; they should not be silently substituted.
103
 
104
  Earlier internal diagnostics and their measurements remain available in the [versioned evaluation record](https://huggingface.co/SlayerLab/Slayer1B-1.08/blob/5f854a0cc7160594d54d1788d37a44e8ebabe16c/README.md). They are historical measurements, not this card's release criteria.