| { |
| "model": "BrainboxAI/nitzotz", |
| "eval_set": "8 frozen suites, 3990 questions (checksums in the repo's eval/manifest.json)", |
| "models": { |
| "nz": "BrainboxAI/nitzotz", |
| "roeig": "RoeiG/laya-hebrew", |
| "laya_ml": "convaiinnovations/laya (multilingual)" |
| }, |
| "licences": { |
| "nz": "Apache-2.0", |
| "roeig": "CC-BY-NC-SA-4.0", |
| "laya_ml": "Apache-2.0" |
| }, |
| "suites": { |
| "spam_scam": { |
| "title": "Scam or not? (298 messages)", |
| "n": 298, |
| "chance": 0.5, |
| "acc": { |
| "nz": 0.9195, |
| "roeig": 0.3221, |
| "laya_ml": 0.5034 |
| }, |
| "brier": { |
| "nz": 0.1321, |
| "roeig": 0.7604, |
| "laya_ml": 0.8042 |
| }, |
| "ece": { |
| "nz": 0.0714, |
| "roeig": 0.3802, |
| "laya_ml": 0.38 |
| }, |
| "empty_acc": { |
| "nz": 0.7047, |
| "roeig": 0.2953, |
| "laya_ml": 0.2953 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 186, |
| "other_only": 8, |
| "p": 3.5766488810898616e-45 |
| }, |
| "laya_ml": { |
| "nz_only": 138, |
| "other_only": 14, |
| "p": 8.462475337125398e-27 |
| } |
| } |
| }, |
| "spam_scam_hard": { |
| "title": "same, hard cases only (65)", |
| "n": 65, |
| "chance": 0.5, |
| "acc": { |
| "nz": 0.8308, |
| "roeig": 0.3231, |
| "laya_ml": 0.3538 |
| }, |
| "brier": { |
| "nz": 0.2842, |
| "roeig": 0.7076, |
| "laya_ml": 1.1057 |
| }, |
| "ece": { |
| "nz": 0.1132, |
| "roeig": 0.3391, |
| "laya_ml": 0.5564 |
| }, |
| "empty_acc": { |
| "nz": 0.7077, |
| "roeig": 0.2923, |
| "laya_ml": 0.2923 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 34, |
| "other_only": 1, |
| "p": 2.0954757928848267e-09 |
| }, |
| "laya_ml": { |
| "nz_only": 35, |
| "other_only": 4, |
| "p": 3.353161446284503e-07 |
| } |
| } |
| }, |
| "spam_type": { |
| "title": "Message type, 6 options (298)", |
| "n": 298, |
| "chance": 0.1667, |
| "acc": { |
| "nz": 0.7785, |
| "roeig": 0.5268, |
| "laya_ml": 0.2081 |
| }, |
| "brier": { |
| "nz": 0.3312, |
| "roeig": 0.6489, |
| "laya_ml": 1.0746 |
| }, |
| "ece": { |
| "nz": 0.0683, |
| "roeig": 0.1043, |
| "laya_ml": 0.4131 |
| }, |
| "empty_acc": { |
| "nz": 0.0772, |
| "roeig": 0.2013, |
| "laya_ml": 0.0772 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 108, |
| "other_only": 33, |
| "p": 1.6862794099851315e-10 |
| }, |
| "laya_ml": { |
| "nz_only": 177, |
| "other_only": 7, |
| "p": 1.0714267651676255e-43 |
| } |
| } |
| }, |
| "spam_type_hard": { |
| "title": "same, hard cases only (65)", |
| "n": 65, |
| "chance": 0.1667, |
| "acc": { |
| "nz": 0.6769, |
| "roeig": 0.4462, |
| "laya_ml": 0.2462 |
| }, |
| "brier": { |
| "nz": 0.4865, |
| "roeig": 0.7315, |
| "laya_ml": 1.1094 |
| }, |
| "ece": { |
| "nz": 0.1027, |
| "roeig": 0.1877, |
| "laya_ml": 0.4109 |
| }, |
| "empty_acc": { |
| "nz": 0.1385, |
| "roeig": 0.1538, |
| "laya_ml": 0.1385 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 23, |
| "other_only": 8, |
| "p": 0.010673840530216694 |
| }, |
| "laya_ml": { |
| "nz_only": 31, |
| "other_only": 3, |
| "p": 7.660128176212311e-07 |
| } |
| } |
| }, |
| "triage_type": { |
| "title": "Support ticket type, 5 options (30)", |
| "n": 30, |
| "chance": 0.2, |
| "acc": { |
| "nz": 0.9, |
| "roeig": 0.8, |
| "laya_ml": 0.5667 |
| }, |
| "brier": { |
| "nz": 0.1838, |
| "roeig": 0.3191, |
| "laya_ml": 0.6667 |
| }, |
| "ece": { |
| "nz": 0.1292, |
| "roeig": 0.1321, |
| "laya_ml": 0.2772 |
| }, |
| "empty_acc": { |
| "nz": 0.1667, |
| "roeig": 0.1667, |
| "laya_ml": 0.3333 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 5, |
| "other_only": 2, |
| "p": 0.453125 |
| }, |
| "laya_ml": { |
| "nz_only": 12, |
| "other_only": 2, |
| "p": 0.012939453125 |
| } |
| } |
| }, |
| "triage_urgency": { |
| "title": "Ticket urgency, 5 levels (30)", |
| "n": 30, |
| "chance": 0.2, |
| "acc": { |
| "nz": 0.7, |
| "roeig": 0.3667, |
| "laya_ml": 0.3333 |
| }, |
| "brier": { |
| "nz": 0.4868, |
| "roeig": 0.708, |
| "laya_ml": 0.7501 |
| }, |
| "ece": { |
| "nz": 0.2041, |
| "roeig": 0.0322, |
| "laya_ml": 0.2167 |
| }, |
| "empty_acc": { |
| "nz": 0.4, |
| "roeig": 0.2333, |
| "laya_ml": 0.0667 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 14, |
| "other_only": 4, |
| "p": 0.0308837890625 |
| }, |
| "laya_ml": { |
| "nz_only": 15, |
| "other_only": 4, |
| "p": 0.0192108154296875 |
| } |
| } |
| }, |
| "triage_paying": { |
| "title": "Paying customer? yes/no (30)", |
| "n": 30, |
| "chance": 0.5, |
| "acc": { |
| "nz": 0.8, |
| "roeig": 0.5667, |
| "laya_ml": 0.5 |
| }, |
| "brier": { |
| "nz": 0.2841, |
| "roeig": 0.4338, |
| "laya_ml": 0.8183 |
| }, |
| "ece": { |
| "nz": 0.1466, |
| "roeig": 0.2362, |
| "laya_ml": 0.4284 |
| }, |
| "empty_acc": { |
| "nz": 0.5333, |
| "roeig": 0.5333, |
| "laya_ml": 0.4667 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 7, |
| "other_only": 0, |
| "p": 0.015625 |
| }, |
| "laya_ml": { |
| "nz_only": 14, |
| "other_only": 5, |
| "p": 0.063568115234375 |
| } |
| } |
| }, |
| "massive_20": { |
| "title": "Voice command intent, 20 options (500)", |
| "n": 500, |
| "chance": 0.05, |
| "acc": { |
| "nz": 0.9, |
| "roeig": 0.726, |
| "laya_ml": 0.474 |
| }, |
| "brier": { |
| "nz": 0.1671, |
| "roeig": 0.3801, |
| "laya_ml": 0.7338 |
| }, |
| "ece": { |
| "nz": 0.0583, |
| "roeig": 0.0395, |
| "laya_ml": 0.1661 |
| }, |
| "empty_acc": { |
| "nz": 0.064, |
| "roeig": 0.052, |
| "laya_ml": 0.044 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 98, |
| "other_only": 11, |
| "p": 1.3281749873159828e-18 |
| }, |
| "laya_ml": { |
| "nz_only": 222, |
| "other_only": 9, |
| "p": 2.661469004156119e-54 |
| } |
| } |
| }, |
| "massive_4": { |
| "title": "Voice command intent, 4 options (500)", |
| "n": 500, |
| "chance": 0.25, |
| "acc": { |
| "nz": 0.972, |
| "roeig": 0.908, |
| "laya_ml": 0.694 |
| }, |
| "brier": { |
| "nz": 0.0462, |
| "roeig": 0.1369, |
| "laya_ml": 0.3987 |
| }, |
| "ece": { |
| "nz": 0.0501, |
| "roeig": 0.0393, |
| "laya_ml": 0.0468 |
| }, |
| "empty_acc": { |
| "nz": 0.254, |
| "roeig": 0.242, |
| "laya_ml": 0.256 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 35, |
| "other_only": 3, |
| "p": 6.677873898297548e-08 |
| }, |
| "laya_ml": { |
| "nz_only": 141, |
| "other_only": 2, |
| "p": 1.846933796755538e-39 |
| } |
| } |
| }, |
| "sib200": { |
| "title": "News topic, 7 options (204)", |
| "n": 204, |
| "chance": 0.1429, |
| "acc": { |
| "nz": 0.7941, |
| "roeig": 0.8235, |
| "laya_ml": 0.6618 |
| }, |
| "brier": { |
| "nz": 0.3027, |
| "roeig": 0.285, |
| "laya_ml": 0.4918 |
| }, |
| "ece": { |
| "nz": 0.0708, |
| "roeig": 0.0751, |
| "laya_ml": 0.1784 |
| }, |
| "empty_acc": { |
| "nz": 0.1471, |
| "roeig": 0.1961, |
| "laya_ml": 0.25 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 12, |
| "other_only": 18, |
| "p": 0.361594608053565 |
| }, |
| "laya_ml": { |
| "nz_only": 47, |
| "other_only": 20, |
| "p": 0.0013071686857636005 |
| } |
| } |
| }, |
| "heq_verify": { |
| "title": "Does the passage support this answer? (600)", |
| "n": 600, |
| "chance": 0.5, |
| "acc": { |
| "nz": 0.9483, |
| "roeig": 0.9417, |
| "laya_ml": 0.4883 |
| }, |
| "brier": { |
| "nz": 0.0938, |
| "roeig": 0.0926, |
| "laya_ml": 0.8274 |
| }, |
| "ece": { |
| "nz": 0.0365, |
| "roeig": 0.0178, |
| "laya_ml": 0.3946 |
| }, |
| "empty_acc": { |
| "nz": 0.51, |
| "roeig": 0.8, |
| "laya_ml": 0.59 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 30, |
| "other_only": 26, |
| "p": 0.6888797607233396 |
| }, |
| "laya_ml": { |
| "nz_only": 296, |
| "other_only": 20, |
| "p": 3.520918733970021e-64 |
| } |
| } |
| }, |
| "heq_unans": { |
| "title": "Plausible answer the passage does not give (600)", |
| "n": 600, |
| "chance": 0.5, |
| "acc": { |
| "nz": 0.8917, |
| "roeig": 0.53, |
| "laya_ml": 0.5367 |
| }, |
| "brier": { |
| "nz": 0.1739, |
| "roeig": 0.856, |
| "laya_ml": 0.7348 |
| }, |
| "ece": { |
| "nz": 0.0484, |
| "roeig": 0.4336, |
| "laya_ml": 0.3421 |
| }, |
| "empty_acc": { |
| "nz": 0.5033, |
| "roeig": 0.5033, |
| "laya_ml": 0.5167 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 243, |
| "other_only": 26, |
| "p": 2.502982180208191e-45 |
| }, |
| "laya_ml": { |
| "nz_only": 240, |
| "other_only": 27, |
| "p": 7.334224519429334e-44 |
| } |
| } |
| }, |
| "belebele": { |
| "title": "Reading comprehension, 4 options (900)", |
| "n": 900, |
| "chance": 0.25, |
| "acc": { |
| "nz": 0.6322, |
| "roeig": 0.7544, |
| "laya_ml": 0.3144 |
| }, |
| "brier": { |
| "nz": 0.5672, |
| "roeig": 0.3442, |
| "laya_ml": 0.8341 |
| }, |
| "ece": { |
| "nz": 0.1919, |
| "roeig": 0.0324, |
| "laya_ml": 0.2307 |
| }, |
| "empty_acc": { |
| "nz": 0.3011, |
| "roeig": 0.3189, |
| "laya_ml": 0.2844 |
| }, |
| "vs_nitzotz": { |
| "roeig": { |
| "nz_only": 72, |
| "other_only": 182, |
| "p": 3.66511528586527e-12 |
| }, |
| "laya_ml": { |
| "nz_only": 366, |
| "other_only": 80, |
| "p": 9.158467645861244e-45 |
| } |
| } |
| } |
| }, |
| "significance": "paired exact McNemar against Nitzotz on the same questions", |
| "latency_ms": { |
| "q8": { |
| "median": 47.7, |
| "p90": 75.6, |
| "n": 50 |
| }, |
| "f16": { |
| "median": 50.5, |
| "p90": 77.0, |
| "n": 50 |
| }, |
| "py": { |
| "median": 60.0, |
| "p90": 98.2, |
| "n": 50 |
| }, |
| "q8_cpu": { |
| "median": 309.1, |
| "p90": 711.0, |
| "n": 50 |
| }, |
| "f16_cpu": { |
| "median": 425.9, |
| "p90": 854.0, |
| "n": 50 |
| }, |
| "py_cpu": { |
| "median": 223.9, |
| "p90": 452.7, |
| "n": 50 |
| } |
| }, |
| "latency_points_ms": { |
| "q8": { |
| "short": { |
| "item": "massive_he_4-0075", |
| "input_tokens": 38, |
| "median_ms": 36.9, |
| "p90_ms": 38.4, |
| "min_ms": 35.6 |
| }, |
| "long": { |
| "item": "heq_unanswerable-0522", |
| "input_tokens": 430, |
| "median_ms": 127.4, |
| "p90_ms": 130.0, |
| "min_ms": 108.0 |
| } |
| }, |
| "f16": { |
| "short": { |
| "item": "massive_he_4-0075", |
| "input_tokens": 38, |
| "median_ms": 43.2, |
| "p90_ms": 44.3, |
| "min_ms": 41.2 |
| }, |
| "long": { |
| "item": "heq_unanswerable-0522", |
| "input_tokens": 430, |
| "median_ms": 111.2, |
| "p90_ms": 118.2, |
| "min_ms": 108.7 |
| } |
| }, |
| "py": { |
| "short": { |
| "item": "massive_he_4-0075", |
| "input_tokens": 38, |
| "median_ms": 43.6, |
| "p90_ms": 44.7, |
| "min_ms": 42.4 |
| }, |
| "long": { |
| "item": "heq_unanswerable-0522", |
| "input_tokens": 430, |
| "median_ms": 139.3, |
| "p90_ms": 157.0, |
| "min_ms": 137.1 |
| } |
| }, |
| "q8_cpu": { |
| "short": { |
| "item": "massive_he_4-0075", |
| "input_tokens": 38, |
| "median_ms": 159.5, |
| "p90_ms": 185.0, |
| "min_ms": 117.5 |
| }, |
| "long": { |
| "item": "heq_unanswerable-0522", |
| "input_tokens": 430, |
| "median_ms": 1137.4, |
| "p90_ms": 1243.6, |
| "min_ms": 1072.4 |
| } |
| }, |
| "py_cpu": { |
| "short": { |
| "item": "massive_he_4-0075", |
| "input_tokens": 38, |
| "median_ms": 112.0, |
| "p90_ms": 131.3, |
| "min_ms": 97.1 |
| }, |
| "long": { |
| "item": "heq_unanswerable-0522", |
| "input_tokens": 430, |
| "median_ms": 1021.8, |
| "p90_ms": 1100.7, |
| "min_ms": 918.8 |
| } |
| } |
| }, |
| "latency_machine": "Intel Core Ultra 9 285H laptop, built-in Arc 140T GPU, Windows 11", |
| "gguf_parity_50": { |
| "q8": { |
| "max_abs_diff": 0.0375, |
| "mean_abs_diff": 0.001814, |
| "argmax_agree": "50/50" |
| }, |
| "f16": { |
| "max_abs_diff": 0.0017, |
| "mean_abs_diff": 0.000288, |
| "argmax_agree": "50/50" |
| }, |
| "q8_cpu": { |
| "max_abs_diff": 0.0549, |
| "mean_abs_diff": 0.003498, |
| "argmax_agree": "50/50" |
| }, |
| "f16_cpu": { |
| "max_abs_diff": 0.0032, |
| "mean_abs_diff": 0.000278, |
| "argmax_agree": "50/50" |
| } |
| }, |
| "gguf_parity_spam_596": { |
| "q8": { |
| "max_abs_diff": 0.0166, |
| "mean_abs_diff": 0.001297, |
| "argmax_agree": "596/596", |
| "accuracy_vs_gold": 0.8473 |
| }, |
| "f16": { |
| "max_abs_diff": 0.0015, |
| "mean_abs_diff": 0.000203, |
| "argmax_agree": "596/596", |
| "accuracy_vs_gold": 0.8473 |
| }, |
| "py": { |
| "max_abs_diff": 0.0, |
| "mean_abs_diff": 0.0, |
| "argmax_agree": "596/596", |
| "accuracy_vs_gold": 0.8473 |
| }, |
| "q8_cpu": { |
| "max_abs_diff": 0.0207, |
| "mean_abs_diff": 0.001918, |
| "argmax_agree": "595/596", |
| "accuracy_vs_gold": 0.8456 |
| }, |
| "f16_cpu": { |
| "max_abs_diff": 0.0012, |
| "mean_abs_diff": 0.000185, |
| "argmax_agree": "596/596", |
| "accuracy_vs_gold": 0.8473 |
| }, |
| "py_cpu": { |
| "max_abs_diff": 0.0, |
| "mean_abs_diff": 0.0, |
| "argmax_agree": "596/596", |
| "accuracy_vs_gold": 0.8473 |
| } |
| }, |
| "belebele_split": { |
| "verbatim_n": 252, |
| "reworded_n": 648, |
| "nz": { |
| "verbatim": 0.7658730158730159, |
| "reworded": 0.5802469135802469 |
| }, |
| "roeig": { |
| "verbatim": 0.8412698412698413, |
| "reworded": 0.720679012345679 |
| }, |
| "laya_ml": { |
| "verbatim": 0.44047619047619047, |
| "reworded": 0.2654320987654321 |
| } |
| }, |
| "calibration_pooled": { |
| "n": 3990, |
| "ece_15bin": 0.0309 |
| }, |
| "wording_robustness": { |
| "suite": "wording_he (frozen): 1786 questions, each asked in 6 wordings with the same meaning", |
| "flip_rate": "share of questions whose top answer is not the same in all 6 wordings (lower is better)", |
| "models": { |
| "BrainboxAI/nitzotz": { |
| "all_item_weighted": { |
| "flip_rate": 0.0504, |
| "flip_vs_orig": 0.018, |
| "mean_acc": 0.9118, |
| "worst_acc": 0.9048, |
| "flip_rate_spam_type_without_form_swap": 0.0481 |
| }, |
| "parts": { |
| "heq_verify": { |
| "n_items": 600, |
| "mean_acc": 0.9478, |
| "worst_acc": 0.945, |
| "flip_rate": 0.0283 |
| }, |
| "massive_he_4": { |
| "n_items": 500, |
| "mean_acc": 0.97, |
| "worst_acc": 0.962, |
| "flip_rate": 0.036 |
| }, |
| "spam_business_he:category": { |
| "n_items": 298, |
| "mean_acc": 0.7752, |
| "worst_acc": 0.7685, |
| "flip_rate": 0.0805 |
| }, |
| "spam_business_he:category:hard": { |
| "n_items": 65, |
| "mean_acc": 0.6692, |
| "worst_acc": 0.6462, |
| "flip_rate": 0.1077 |
| }, |
| "spam_business_he:scam": { |
| "n_items": 298, |
| "mean_acc": 0.915, |
| "worst_acc": 0.9128, |
| "flip_rate": 0.0201 |
| }, |
| "spam_business_he:scam:hard": { |
| "n_items": 65, |
| "mean_acc": 0.8154, |
| "worst_acc": 0.8, |
| "flip_rate": 0.0615 |
| }, |
| "triage30:category": { |
| "n_items": 30, |
| "mean_acc": 0.9166, |
| "worst_acc": 0.9, |
| "flip_rate": 0.0333 |
| }, |
| "triage30:paying": { |
| "n_items": 30, |
| "mean_acc": 0.7944, |
| "worst_acc": 0.7333, |
| "flip_rate": 0.1667 |
| }, |
| "triage30:urgency": { |
| "n_items": 30, |
| "mean_acc": 0.6611, |
| "worst_acc": 0.6, |
| "flip_rate": 0.6333 |
| } |
| }, |
| "single_answer_wordings": {} |
| } |
| } |
| }, |
| "belebele_negation": { |
| "n": 158, |
| "acc": 0.5886 |
| }, |
| "seeds": { |
| "heldout_this": 0.9505, |
| "others": [ |
| { |
| "name": "seed 2", |
| "heldout": 0.9519, |
| "acc": { |
| "spam_scam": 0.9195, |
| "spam_scam_hard": 0.8308, |
| "spam_type": 0.802, |
| "spam_type_hard": 0.7077, |
| "triage_type": 0.9, |
| "triage_urgency": 0.7333, |
| "triage_paying": 0.8, |
| "massive_20": 0.894, |
| "massive_4": 0.96, |
| "sib200": 0.8284, |
| "heq_verify": 0.9467, |
| "heq_unans": 0.8983, |
| "belebele": 0.6089 |
| } |
| }, |
| { |
| "name": "seed 3", |
| "heldout": 0.9521, |
| "acc": { |
| "spam_scam": 0.906, |
| "spam_scam_hard": 0.7846, |
| "spam_type": 0.7785, |
| "spam_type_hard": 0.6615, |
| "triage_type": 0.8333, |
| "triage_urgency": 0.7667, |
| "triage_paying": 0.8333, |
| "massive_20": 0.902, |
| "massive_4": 0.968, |
| "sib200": 0.8039, |
| "heq_verify": 0.95, |
| "heq_unans": 0.8983, |
| "belebele": 0.6311 |
| } |
| } |
| ], |
| "agreement_min": 0.9008, |
| "agreement_max": 0.9063, |
| "picked_on": { |
| "this": { |
| "n": 188, |
| "acc": 0.984 |
| }, |
| "others": [ |
| { |
| "n": 188, |
| "acc": 0.9734 |
| }, |
| { |
| "n": 188, |
| "acc": 0.9734 |
| } |
| ] |
| } |
| }, |
| "none_of_the_options": { |
| "suite": "notinlist_he (frozen): 400 MASSIVE and Belebele test questions with a 'none of the options' choice added; in half of them 'none' is right, in the other half the right answer is still in the list", |
| "model": "BrainboxAI/nitzotz", |
| "parts": { |
| "all": { |
| "n": 400, |
| "acc": 0.685, |
| "picks_none": 0.535 |
| }, |
| "belebele_he/control": { |
| "n": 100, |
| "acc": 0.45, |
| "picks_none": 0.43 |
| }, |
| "belebele_he/none": { |
| "n": 100, |
| "acc": 0.76, |
| "picks_none": 0.76 |
| }, |
| "control": { |
| "n": 200, |
| "acc": 0.61, |
| "picks_none": 0.31 |
| }, |
| "massive_he_20/control": { |
| "n": 100, |
| "acc": 0.77, |
| "picks_none": 0.19 |
| }, |
| "massive_he_20/none": { |
| "n": 100, |
| "acc": 0.76, |
| "picks_none": 0.76 |
| }, |
| "none": { |
| "n": 200, |
| "acc": 0.76, |
| "picks_none": 0.76 |
| } |
| }, |
| "text_removed": { |
| "all": { |
| "n": 400, |
| "acc": 0.49, |
| "picks_none": 0.9825 |
| }, |
| "belebele_he/control": { |
| "n": 100, |
| "acc": 0.01, |
| "picks_none": 0.99 |
| }, |
| "belebele_he/none": { |
| "n": 100, |
| "acc": 0.95, |
| "picks_none": 0.95 |
| }, |
| "control": { |
| "n": 200, |
| "acc": 0.005, |
| "picks_none": 0.99 |
| }, |
| "massive_he_20/control": { |
| "n": 100, |
| "acc": 0.0, |
| "picks_none": 0.99 |
| }, |
| "massive_he_20/none": { |
| "n": 100, |
| "acc": 1.0, |
| "picks_none": 1.0 |
| }, |
| "none": { |
| "n": 200, |
| "acc": 0.975, |
| "picks_none": 0.975 |
| } |
| } |
| } |
| } |
