| { |
| "lineage": [ |
| { |
| "stage": "pretrain", |
| "run_id": "1p6b-pretrain-v3", |
| "parent_run_id": null, |
| "parent_weights_sha256": null, |
| "weights_sha256": "dd3a794d333863035bbe74547a7a310ead79cdc282120c694c49133262d957f9", |
| "global_step_at_save": 152588, |
| "tokens_seen": 9960847877, |
| "training_recipe_hash": "b4ff73d394fba440", |
| "dataset_subset_id": "", |
| "code_commit": "e723ba145652a3092aea457e498abd8cca27a6a7", |
| "config_seed": 1234, |
| "saved_at": "2026-09-24T02:13:41", |
| "selected_epoch": null, |
| "val_score": null |
| }, |
| { |
| "stage": "v2", |
| "run_id": "decisions-v2-e8", |
| "parent_run_id": "1p6b-pretrain-v3", |
| "parent_weights_sha256": "dd3a794d333863035bbe74547a7a310ead79cdc282120c694c49133262d957f9", |
| "weights_sha256": "d5c76527a9b03c8315df8af7abb55acbfa9ab811c68f55bbd057bf2b796c8fbf", |
| "global_step_at_save": 10120, |
| "tokens_seen": 0, |
| "training_recipe_hash": "1c87e80c550a9e36", |
| "dataset_subset_id": "typed_train_en+typed_train_es+jev_train", |
| "code_commit": "5fd2f4180d321e69838dd43b6f1cec6da261064a", |
| "config_seed": 1234, |
| "saved_at": "2026-09-30T20:09:38", |
| "selected_epoch": 7, |
| "val_score": 0.6914998056527292, |
| "train_files": [ |
| "typed_train_en.jsonl", |
| "typed_train_es.jsonl", |
| "jev_train.jsonl" |
| ], |
| "val_files": [ |
| "typed_val_en.jsonl", |
| "typed_val_es.jsonl", |
| "jev_val.jsonl" |
| ], |
| "epochs_scheduled": 8, |
| "lr": 1e-05, |
| "accum": 16, |
| "aux": 0.01, |
| "sft_seed": 7, |
| "val_curve": [ |
| 0.3706, |
| 0.4997, |
| 0.601, |
| 0.6721, |
| 0.6709, |
| 0.675, |
| 0.6915, |
| 0.6901 |
| ], |
| "train_questions_per_epoch": 20233, |
| "optimizer_steps_scheduled": 10117, |
| "optimizer_steps_to_selected_weights": 8852 |
| }, |
| { |
| "stage": "v3", |
| "run_id": "decisions-v3-mix-from-v2", |
| "parent_run_id": "decisions-v2-e8", |
| "parent_weights_sha256": "d5c76527a9b03c8315df8af7abb55acbfa9ab811c68f55bbd057bf2b796c8fbf", |
| "weights_sha256": "5657d3ee7bf9934a52a34f270aa4ebc78ae69ef788ad49c0b07cbd1fe8ebe8fb", |
| "global_step_at_save": 10806, |
| "tokens_seen": 0, |
| "training_recipe_hash": "e4b355103247ef81", |
| "dataset_subset_id": "train_helmo+train_jev+train_n4ze3m+train_tasksource+train_typed_en+train_typed_es+train_typed_es_telepatia", |
| "code_commit": "0b5e63e3da05c1a51eb9f0ea1758f8d53d200a80-dirty", |
| "config_seed": 1234, |
| "saved_at": "2026-10-01T13:45:06", |
| "selected_epoch": 2, |
| "val_score": 0.6444873254407529, |
| "train_files": [ |
| "train_helmo.jsonl", |
| "train_jev.jsonl", |
| "train_n4ze3m.jsonl", |
| "train_tasksource.jsonl", |
| "train_typed_en.jsonl", |
| "train_typed_es.jsonl", |
| "train_typed_es_telepatia.jsonl" |
| ], |
| "val_files": [ |
| "val_helmo.jsonl", |
| "val_jev.jsonl", |
| "val_n4ze3m.jsonl", |
| "val_tasksource.jsonl", |
| "val_typed_en.jsonl", |
| "val_typed_es.jsonl" |
| ], |
| "epochs_scheduled": 2, |
| "lr": 1e-05, |
| "accum": 16, |
| "aux": 0.01, |
| "sft_seed": 7, |
| "val_curve": [ |
| 0.6276, |
| 0.6445 |
| ], |
| "train_questions_per_epoch": 86436, |
| "optimizer_steps_scheduled": 10805, |
| "optimizer_steps_to_selected_weights": 10805 |
| }, |
| { |
| "stage": "v3pre", |
| "run_id": "decisions-v3-mix", |
| "parent_run_id": "1p6b-pretrain-v3", |
| "parent_weights_sha256": "dd3a794d333863035bbe74547a7a310ead79cdc282120c694c49133262d957f9", |
| "weights_sha256": "221d0663414de1a754bd28499d355d0e3946836cca772bed8dc069a1fd2b9ce4", |
| "global_step_at_save": 16209, |
| "tokens_seen": 0, |
| "training_recipe_hash": "b5f63c4154bf184e", |
| "dataset_subset_id": "train_helmo+train_jev+train_n4ze3m+train_tasksource+train_typed_en+train_typed_es+train_typed_es_telepatia", |
| "code_commit": "0b5e63e3da05c1a51eb9f0ea1758f8d53d200a80-dirty", |
| "config_seed": 1234, |
| "saved_at": "2026-10-01T15:53:18", |
| "selected_epoch": 3, |
| "val_score": 0.6358126982153586, |
| "train_files": [ |
| "train_helmo.jsonl", |
| "train_jev.jsonl", |
| "train_n4ze3m.jsonl", |
| "train_tasksource.jsonl", |
| "train_typed_en.jsonl", |
| "train_typed_es.jsonl", |
| "train_typed_es_telepatia.jsonl" |
| ], |
| "val_files": [ |
| "val_helmo.jsonl", |
| "val_jev.jsonl", |
| "val_n4ze3m.jsonl", |
| "val_tasksource.jsonl", |
| "val_typed_en.jsonl", |
| "val_typed_es.jsonl" |
| ], |
| "epochs_scheduled": 3, |
| "lr": 1e-05, |
| "accum": 16, |
| "aux": 0.01, |
| "sft_seed": 7, |
| "val_curve": [ |
| 0.539, |
| 0.6185, |
| 0.6358 |
| ], |
| "train_questions_per_epoch": 86436, |
| "optimizer_steps_scheduled": 16207, |
| "optimizer_steps_to_selected_weights": 16207 |
| } |
| ], |
| "reference_rows": { |
| "uniform": { |
| "n": 2000, |
| "correct": 548, |
| "accuracy": 0.274, |
| "nll": 1.2118217396490383, |
| "kl": 0.44446318402470963, |
| "ce_soft": 1.2118217396490383, |
| "brier": 0.2381881657480489, |
| "brier_mean_k": 0.07191006022457894, |
| "brier_onehot": 0.6825, |
| "ece10": 0.048499999999999995, |
| "mean_conf": 0.3175, |
| "by_type": { |
| "choice": { |
| "correct": 119, |
| "n": 600, |
| "accuracy": 0.19833333333333333 |
| }, |
| "noul": { |
| "correct": 305, |
| "n": 600, |
| "accuracy": 0.5083333333333333 |
| }, |
| "score": { |
| "correct": 124, |
| "n": 800, |
| "accuracy": 0.155 |
| } |
| }, |
| "accuracy_expected_1_over_k": 0.3175 |
| }, |
| "prior": { |
| "n": 2000, |
| "correct": 967, |
| "accuracy": 0.4835, |
| "nll": 1.0147031719053614, |
| "kl": 0.4374539124612297, |
| "ce_soft": 1.2048124680855585, |
| "brier": 0.19727670747180912, |
| "brier_mean_k": 0.0579368310718516, |
| "brier_onehot": 0.59002, |
| "ece10": 0.038666666666666655, |
| "mean_conf": 0.5148333333333334, |
| "by_type": { |
| "choice": { |
| "correct": 251, |
| "n": 600, |
| "accuracy": 0.41833333333333333 |
| }, |
| "noul": { |
| "correct": 385, |
| "n": 600, |
| "accuracy": 0.6416666666666667 |
| }, |
| "score": { |
| "correct": 331, |
| "n": 800, |
| "accuracy": 0.41375 |
| } |
| } |
| }, |
| "argmax_agree_counts": { |
| "false": 812, |
| "true": 1188 |
| } |
| }, |
| "results": { |
| "v2|tasksource": { |
| "n": 2206, |
| "correct": 850, |
| "accuracy": 0.38531278331822305, |
| "nll": 1.18249501294161, |
| "kl": 1.1583903767910129, |
| "ce_soft": 1.1835308450719855, |
| "brier": 0.6582691515879918, |
| "brier_mean_k": 0.2238124108887532, |
| "brier_onehot": 0.6728618433752923, |
| "ece10": 0.12647392187378995, |
| "mean_conf": 0.511786705192013, |
| "by_type": { |
| "choice": { |
| "correct": 664, |
| "n": 1815, |
| "accuracy": 0.36584022038567493 |
| }, |
| "noul": { |
| "correct": 151, |
| "n": 261, |
| "accuracy": 0.578544061302682 |
| }, |
| "score": { |
| "correct": 35, |
| "n": 130, |
| "accuracy": 0.2692307692307692 |
| } |
| }, |
| "ci95_cases": [ |
| 0.3653671804170444, |
| 0.4070716228467815 |
| ], |
| "mean_prompt_tokens": 206.149138712602 |
| }, |
| "v2|telepatia_es": { |
| "n": 2000, |
| "correct": 1492, |
| "accuracy": 0.746, |
| "nll": 0.6768666282900727, |
| "kl": 0.137470178826373, |
| "ce_soft": 0.9048287344507019, |
| "brier": 0.07226937229140096, |
| "brier_mean_k": 0.020558110269673965, |
| "brier_onehot": 0.3838231707888498, |
| "ece10": 0.1303335434347391, |
| "mean_conf": 0.6159254869967699, |
| "by_type": { |
| "choice": { |
| "correct": 415, |
| "n": 600, |
| "accuracy": 0.6916666666666667 |
| }, |
| "noul": { |
| "correct": 502, |
| "n": 600, |
| "accuracy": 0.8366666666666667 |
| }, |
| "score": { |
| "correct": 575, |
| "n": 800, |
| "accuracy": 0.71875 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7245, |
| 0.768 |
| ], |
| "mean_prompt_tokens": 315.093 |
| }, |
| "v2|typed_en": { |
| "n": 2000, |
| "correct": 1510, |
| "accuracy": 0.755, |
| "nll": 0.6463324192750852, |
| "kl": 0.13270933329166798, |
| "ce_soft": 0.9000678889159969, |
| "brier": 0.07002914815029006, |
| "brier_mean_k": 0.02004525517631372, |
| "brier_onehot": 0.3676075399939686, |
| "ece10": 0.12290431407094005, |
| "mean_conf": 0.6332694019079208, |
| "by_type": { |
| "choice": { |
| "correct": 411, |
| "n": 600, |
| "accuracy": 0.685 |
| }, |
| "noul": { |
| "correct": 503, |
| "n": 600, |
| "accuracy": 0.8383333333333334 |
| }, |
| "score": { |
| "correct": 596, |
| "n": 800, |
| "accuracy": 0.745 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7335, |
| 0.7765 |
| ], |
| "mean_prompt_tokens": 296.0125 |
| }, |
| "v2|typed_es": { |
| "n": 1985, |
| "correct": 1482, |
| "accuracy": 0.7465994962216624, |
| "nll": 0.6550145421374155, |
| "kl": 0.13250811726703088, |
| "ce_soft": 0.9007772998807515, |
| "brier": 0.07028015084314279, |
| "brier_mean_k": 0.01999555773940268, |
| "brier_onehot": 0.3732510221786229, |
| "ece10": 0.1152207864471887, |
| "mean_conf": 0.6313787097744737, |
| "by_type": { |
| "choice": { |
| "correct": 401, |
| "n": 594, |
| "accuracy": 0.6750841750841751 |
| }, |
| "noul": { |
| "correct": 497, |
| "n": 597, |
| "accuracy": 0.8324958123953099 |
| }, |
| "score": { |
| "correct": 584, |
| "n": 794, |
| "accuracy": 0.7355163727959698 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7249370277078085, |
| 0.7677581863979849 |
| ], |
| "mean_prompt_tokens": 317.00705289672544 |
| }, |
| "v2-engram_off|tasksource": { |
| "n": 2206, |
| "correct": 816, |
| "accuracy": 0.36990027198549413, |
| "nll": 1.177764537558081, |
| "kl": 1.1535940646862644, |
| "ce_soft": 1.1787345329672367, |
| "brier": 0.6549310146754723, |
| "brier_mean_k": 0.22274428248490724, |
| "brier_onehot": 0.6696232285301682, |
| "ece10": 0.1318699845221782, |
| "mean_conf": 0.5017702565076723, |
| "by_type": { |
| "choice": { |
| "correct": 634, |
| "n": 1815, |
| "accuracy": 0.3493112947658402 |
| }, |
| "noul": { |
| "correct": 146, |
| "n": 261, |
| "accuracy": 0.5593869731800766 |
| }, |
| "score": { |
| "correct": 36, |
| "n": 130, |
| "accuracy": 0.27692307692307694 |
| } |
| }, |
| "ci95_cases": [ |
| 0.34950135992747056, |
| 0.39029918404351766 |
| ], |
| "mean_prompt_tokens": 206.149138712602 |
| }, |
| "v2-engram_off|telepatia_es": { |
| "n": 2000, |
| "correct": 1433, |
| "accuracy": 0.7165, |
| "nll": 0.7350957449054891, |
| "kl": 0.1574048776542151, |
| "ce_soft": 0.9247634332785439, |
| "brier": 0.08390837700265645, |
| "brier_mean_k": 0.024310100717596178, |
| "brier_onehot": 0.4186372582045989, |
| "ece10": 0.13328185617923735, |
| "mean_conf": 0.5832181438207626, |
| "by_type": { |
| "choice": { |
| "correct": 404, |
| "n": 600, |
| "accuracy": 0.6733333333333333 |
| }, |
| "noul": { |
| "correct": 494, |
| "n": 600, |
| "accuracy": 0.8233333333333334 |
| }, |
| "score": { |
| "correct": 535, |
| "n": 800, |
| "accuracy": 0.66875 |
| } |
| }, |
| "ci95_cases": [ |
| 0.694, |
| 0.7385 |
| ], |
| "mean_prompt_tokens": 315.093 |
| }, |
| "v2-engram_off|typed_en": { |
| "n": 2000, |
| "correct": 1479, |
| "accuracy": 0.7395, |
| "nll": 0.68511629220165, |
| "kl": 0.1402235323363397, |
| "ce_soft": 0.9075820879606685, |
| "brier": 0.07412362822891774, |
| "brier_mean_k": 0.02127965516331085, |
| "brier_onehot": 0.38946567577027386, |
| "ece10": 0.12908081570267677, |
| "mean_conf": 0.6104191842973232, |
| "by_type": { |
| "choice": { |
| "correct": 408, |
| "n": 600, |
| "accuracy": 0.68 |
| }, |
| "noul": { |
| "correct": 500, |
| "n": 600, |
| "accuracy": 0.8333333333333334 |
| }, |
| "score": { |
| "correct": 571, |
| "n": 800, |
| "accuracy": 0.71375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.718, |
| 0.761 |
| ], |
| "mean_prompt_tokens": 296.0125 |
| }, |
| "v2-engram_off|typed_es": { |
| "n": 1985, |
| "correct": 1427, |
| "accuracy": 0.7188916876574307, |
| "nll": 0.7129132213174406, |
| "kl": 0.1504756989477731, |
| "ce_soft": 0.9187448815614937, |
| "brier": 0.08014936250258083, |
| "brier_mean_k": 0.022948228945889947, |
| "brier_onehot": 0.4048794955474447, |
| "ece10": 0.12493646382084421, |
| "mean_conf": 0.6031942302694249, |
| "by_type": { |
| "choice": { |
| "correct": 385, |
| "n": 594, |
| "accuracy": 0.6481481481481481 |
| }, |
| "noul": { |
| "correct": 491, |
| "n": 597, |
| "accuracy": 0.8224455611390284 |
| }, |
| "score": { |
| "correct": 551, |
| "n": 794, |
| "accuracy": 0.6939546599496221 |
| } |
| }, |
| "ci95_cases": [ |
| 0.6972292191435768, |
| 0.7405541561712846 |
| ], |
| "mean_prompt_tokens": 317.00705289672544 |
| }, |
| "v3|tasksource": { |
| "n": 2206, |
| "correct": 1093, |
| "accuracy": 0.49546690843155033, |
| "nll": 1.0302639724427658, |
| "kl": 1.0076541905677538, |
| "ce_soft": 1.0327946588487265, |
| "brier": 0.5674609637187716, |
| "brier_mean_k": 0.19068295958341383, |
| "brier_onehot": 0.5817144817866338, |
| "ece10": 0.07126550326429487, |
| "mean_conf": 0.5667324116958452, |
| "by_type": { |
| "choice": { |
| "correct": 870, |
| "n": 1815, |
| "accuracy": 0.4793388429752066 |
| }, |
| "noul": { |
| "correct": 172, |
| "n": 261, |
| "accuracy": 0.6590038314176245 |
| }, |
| "score": { |
| "correct": 51, |
| "n": 130, |
| "accuracy": 0.3923076923076923 |
| } |
| }, |
| "ci95_cases": [ |
| 0.4741613780598368, |
| 0.5158658204895739 |
| ], |
| "mean_prompt_tokens": 206.149138712602 |
| }, |
| "v3|telepatia_es": { |
| "n": 2000, |
| "correct": 1526, |
| "accuracy": 0.763, |
| "nll": 0.6385563440375813, |
| "kl": 0.11925372995655213, |
| "ce_soft": 0.886612285580881, |
| "brier": 0.06393994905918735, |
| "brier_mean_k": 0.018569114373183912, |
| "brier_onehot": 0.36353173992639604, |
| "ece10": 0.13380355775356292, |
| "mean_conf": 0.6294715915620327, |
| "by_type": { |
| "choice": { |
| "correct": 438, |
| "n": 600, |
| "accuracy": 0.73 |
| }, |
| "noul": { |
| "correct": 498, |
| "n": 600, |
| "accuracy": 0.83 |
| }, |
| "score": { |
| "correct": 590, |
| "n": 800, |
| "accuracy": 0.7375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.742, |
| 0.785 |
| ], |
| "mean_prompt_tokens": 315.093 |
| }, |
| "v3|typed_en": { |
| "n": 2000, |
| "correct": 1528, |
| "accuracy": 0.764, |
| "nll": 0.6348000373513565, |
| "kl": 0.12133635641347341, |
| "ce_soft": 0.8886949120378023, |
| "brier": 0.0651098222513845, |
| "brier_mean_k": 0.019005062125487936, |
| "brier_onehot": 0.3611829084554942, |
| "ece10": 0.13159668615460393, |
| "mean_conf": 0.632403313845396, |
| "by_type": { |
| "choice": { |
| "correct": 429, |
| "n": 600, |
| "accuracy": 0.715 |
| }, |
| "noul": { |
| "correct": 503, |
| "n": 600, |
| "accuracy": 0.8383333333333334 |
| }, |
| "score": { |
| "correct": 596, |
| "n": 800, |
| "accuracy": 0.745 |
| } |
| }, |
| "ci95_cases": [ |
| 0.742, |
| 0.7855 |
| ], |
| "mean_prompt_tokens": 296.0125 |
| }, |
| "v3|typed_es": { |
| "n": 1985, |
| "correct": 1496, |
| "accuracy": 0.7536523929471033, |
| "nll": 0.6394842117466736, |
| "kl": 0.11898906052638147, |
| "ce_soft": 0.887258243140102, |
| "brier": 0.06475655764622247, |
| "brier_mean_k": 0.018860197074895686, |
| "brier_onehot": 0.36670251099112927, |
| "ece10": 0.12293764144107136, |
| "mean_conf": 0.6313106105219507, |
| "by_type": { |
| "choice": { |
| "correct": 415, |
| "n": 594, |
| "accuracy": 0.6986531986531986 |
| }, |
| "noul": { |
| "correct": 494, |
| "n": 597, |
| "accuracy": 0.8274706867671692 |
| }, |
| "score": { |
| "correct": 587, |
| "n": 794, |
| "accuracy": 0.739294710327456 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7324937027707809, |
| 0.7743073047858942 |
| ], |
| "mean_prompt_tokens": 317.00705289672544 |
| }, |
| "v3-engram_off|tasksource": { |
| "n": 2206, |
| "correct": 1037, |
| "accuracy": 0.4700815956482321, |
| "nll": 1.040257259141714, |
| "kl": 1.0165084501963875, |
| "ce_soft": 1.0416489184773599, |
| "brier": 0.5747761215316167, |
| "brier_mean_k": 0.19368823630368673, |
| "brier_onehot": 0.5896047821488928, |
| "ece10": 0.06856633263791244, |
| "mean_conf": 0.5386479282861445, |
| "by_type": { |
| "choice": { |
| "correct": 829, |
| "n": 1815, |
| "accuracy": 0.4567493112947658 |
| }, |
| "noul": { |
| "correct": 156, |
| "n": 261, |
| "accuracy": 0.5977011494252874 |
| }, |
| "score": { |
| "correct": 52, |
| "n": 130, |
| "accuracy": 0.4 |
| } |
| }, |
| "ci95_cases": [ |
| 0.4492293744333635, |
| 0.4904805077062557 |
| ], |
| "mean_prompt_tokens": 206.149138712602 |
| }, |
| "v3-engram_off|telepatia_es": { |
| "n": 2000, |
| "correct": 1469, |
| "accuracy": 0.7345, |
| "nll": 0.6917027070297807, |
| "kl": 0.13656979911128891, |
| "ce_soft": 0.9039283547356177, |
| "brier": 0.07339688864379973, |
| "brier_mean_k": 0.021115876966946143, |
| "brier_onehot": 0.3936445649003366, |
| "ece10": 0.13314792701601982, |
| "mean_conf": 0.6027737509608269, |
| "by_type": { |
| "choice": { |
| "correct": 401, |
| "n": 600, |
| "accuracy": 0.6683333333333333 |
| }, |
| "noul": { |
| "correct": 498, |
| "n": 600, |
| "accuracy": 0.83 |
| }, |
| "score": { |
| "correct": 570, |
| "n": 800, |
| "accuracy": 0.7125 |
| } |
| }, |
| "ci95_cases": [ |
| 0.713, |
| 0.756 |
| ], |
| "mean_prompt_tokens": 315.093 |
| }, |
| "v3-engram_off|typed_en": { |
| "n": 2000, |
| "correct": 1494, |
| "accuracy": 0.747, |
| "nll": 0.6658806168427456, |
| "kl": 0.1298324631061122, |
| "ce_soft": 0.897191018730441, |
| "brier": 0.06997183069647757, |
| "brier_mean_k": 0.0204340936628309, |
| "brier_onehot": 0.3804533671105715, |
| "ece10": 0.13033386017382145, |
| "mean_conf": 0.6169207523316145, |
| "by_type": { |
| "choice": { |
| "correct": 410, |
| "n": 600, |
| "accuracy": 0.6833333333333333 |
| }, |
| "noul": { |
| "correct": 491, |
| "n": 600, |
| "accuracy": 0.8183333333333334 |
| }, |
| "score": { |
| "correct": 593, |
| "n": 800, |
| "accuracy": 0.74125 |
| } |
| }, |
| "ci95_cases": [ |
| 0.726, |
| 0.768 |
| ], |
| "mean_prompt_tokens": 296.0125 |
| }, |
| "v3-engram_off|typed_es": { |
| "n": 1985, |
| "correct": 1446, |
| "accuracy": 0.728463476070529, |
| "nll": 0.6957014567741066, |
| "kl": 0.13709141764197266, |
| "ce_soft": 0.9053606002556932, |
| "brier": 0.07422417302252052, |
| "brier_mean_k": 0.021329935813320985, |
| "brier_onehot": 0.3961568726644083, |
| "ece10": 0.12043466251202734, |
| "mean_conf": 0.6082240724293351, |
| "by_type": { |
| "choice": { |
| "correct": 394, |
| "n": 594, |
| "accuracy": 0.6632996632996633 |
| }, |
| "noul": { |
| "correct": 492, |
| "n": 597, |
| "accuracy": 0.8241206030150754 |
| }, |
| "score": { |
| "correct": 560, |
| "n": 794, |
| "accuracy": 0.7052896725440806 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7062972292191436, |
| 0.7496221662468514 |
| ], |
| "mean_prompt_tokens": 317.00705289672544 |
| }, |
| "v3pre|tasksource": { |
| "n": 2206, |
| "correct": 1118, |
| "accuracy": 0.5067996373526745, |
| "nll": 1.0126227101604877, |
| "kl": 0.989910359101978, |
| "ce_soft": 1.0150508273829506, |
| "brier": 0.5577300772285442, |
| "brier_mean_k": 0.18832110523426265, |
| "brier_onehot": 0.5721825731599312, |
| "ece10": 0.06798556558483856, |
| "mean_conf": 0.5731623702136258, |
| "by_type": { |
| "choice": { |
| "correct": 893, |
| "n": 1815, |
| "accuracy": 0.4920110192837466 |
| }, |
| "noul": { |
| "correct": 167, |
| "n": 261, |
| "accuracy": 0.6398467432950191 |
| }, |
| "score": { |
| "correct": 58, |
| "n": 130, |
| "accuracy": 0.4461538461538462 |
| } |
| }, |
| "ci95_cases": [ |
| 0.485494106980961, |
| 0.5271985494106981 |
| ], |
| "mean_prompt_tokens": 206.149138712602 |
| }, |
| "v3pre|telepatia_es": { |
| "n": 2000, |
| "correct": 1423, |
| "accuracy": 0.7115, |
| "nll": 0.7221613642472114, |
| "kl": 0.16424527027114952, |
| "ce_soft": 0.9316038258954784, |
| "brier": 0.09163500709581687, |
| "brier_mean_k": 0.027993557290275604, |
| "brier_onehot": 0.4151352846821021, |
| "ece10": 0.10952522042393688, |
| "mean_conf": 0.6024507610797882, |
| "by_type": { |
| "choice": { |
| "correct": 381, |
| "n": 600, |
| "accuracy": 0.635 |
| }, |
| "noul": { |
| "correct": 473, |
| "n": 600, |
| "accuracy": 0.7883333333333333 |
| }, |
| "score": { |
| "correct": 569, |
| "n": 800, |
| "accuracy": 0.71125 |
| } |
| }, |
| "ci95_cases": [ |
| 0.69, |
| 0.7315 |
| ], |
| "mean_prompt_tokens": 315.093 |
| }, |
| "v3pre|typed_en": { |
| "n": 2000, |
| "correct": 1451, |
| "accuracy": 0.7255, |
| "nll": 0.7228620920358019, |
| "kl": 0.15801520740611583, |
| "ce_soft": 0.9253737630304447, |
| "brier": 0.08732942362644587, |
| "brier_mean_k": 0.026559532585552777, |
| "brier_onehot": 0.41381279256051157, |
| "ece10": 0.13104377594590189, |
| "mean_conf": 0.5947328799664974, |
| "by_type": { |
| "choice": { |
| "correct": 393, |
| "n": 600, |
| "accuracy": 0.655 |
| }, |
| "noul": { |
| "correct": 487, |
| "n": 600, |
| "accuracy": 0.8116666666666666 |
| }, |
| "score": { |
| "correct": 571, |
| "n": 800, |
| "accuracy": 0.71375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7045, |
| 0.7455 |
| ], |
| "mean_prompt_tokens": 296.0125 |
| }, |
| "v3pre|typed_es": { |
| "n": 1985, |
| "correct": 1397, |
| "accuracy": 0.7037783375314861, |
| "nll": 0.7238645309775039, |
| "kl": 0.16208690588450078, |
| "ce_soft": 0.9303560884982214, |
| "brier": 0.09048640592028294, |
| "brier_mean_k": 0.027522013017270756, |
| "brier_onehot": 0.41629011479103895, |
| "ece10": 0.10134872849702231, |
| "mean_conf": 0.6027212248340962, |
| "by_type": { |
| "choice": { |
| "correct": 365, |
| "n": 594, |
| "accuracy": 0.6144781144781145 |
| }, |
| "noul": { |
| "correct": 475, |
| "n": 597, |
| "accuracy": 0.7956448911222781 |
| }, |
| "score": { |
| "correct": 557, |
| "n": 794, |
| "accuracy": 0.7015113350125944 |
| } |
| }, |
| "ci95_cases": [ |
| 0.6821158690176322, |
| 0.7249370277078085 |
| ], |
| "mean_prompt_tokens": 317.00705289672544 |
| }, |
| "v3pre-engram_off|tasksource": { |
| "n": 2206, |
| "correct": 1083, |
| "accuracy": 0.4909338168631006, |
| "nll": 1.0360851266471505, |
| "kl": 1.0127516723754644, |
| "ce_soft": 1.037892140656437, |
| "brier": 0.5716110348043713, |
| "brier_mean_k": 0.1933716392928341, |
| "brier_onehot": 0.5863590220270787, |
| "ece10": 0.07097867969170227, |
| "mean_conf": 0.5581544994504097, |
| "by_type": { |
| "choice": { |
| "correct": 858, |
| "n": 1815, |
| "accuracy": 0.4727272727272727 |
| }, |
| "noul": { |
| "correct": 172, |
| "n": 261, |
| "accuracy": 0.6590038314176245 |
| }, |
| "score": { |
| "correct": 53, |
| "n": 130, |
| "accuracy": 0.4076923076923077 |
| } |
| }, |
| "ci95_cases": [ |
| 0.4700815956482321, |
| 0.5117860380779692 |
| ], |
| "mean_prompt_tokens": 206.149138712602 |
| }, |
| "v3pre-engram_off|telepatia_es": { |
| "n": 2000, |
| "correct": 1369, |
| "accuracy": 0.6845, |
| "nll": 0.7664083850685155, |
| "kl": 0.1835608149642982, |
| "ce_soft": 0.9509193705886271, |
| "brier": 0.1026449902821665, |
| "brier_mean_k": 0.0314341176817405, |
| "brier_onehot": 0.44305319561626966, |
| "ece10": 0.1018519453555346, |
| "mean_conf": 0.5826480546444655, |
| "by_type": { |
| "choice": { |
| "correct": 376, |
| "n": 600, |
| "accuracy": 0.6266666666666667 |
| }, |
| "noul": { |
| "correct": 462, |
| "n": 600, |
| "accuracy": 0.77 |
| }, |
| "score": { |
| "correct": 531, |
| "n": 800, |
| "accuracy": 0.66375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.6635, |
| 0.706 |
| ], |
| "mean_prompt_tokens": 315.093 |
| }, |
| "v3pre-engram_off|typed_en": { |
| "n": 2000, |
| "correct": 1366, |
| "accuracy": 0.683, |
| "nll": 0.7700775281040052, |
| "kl": 0.18422638726523183, |
| "ce_soft": 0.9515849428895607, |
| "brier": 0.10380722453958674, |
| "brier_mean_k": 0.03158302297002527, |
| "brier_onehot": 0.44732844112781456, |
| "ece10": 0.1008905218988657, |
| "mean_conf": 0.582200200304389, |
| "by_type": { |
| "choice": { |
| "correct": 381, |
| "n": 600, |
| "accuracy": 0.635 |
| }, |
| "noul": { |
| "correct": 462, |
| "n": 600, |
| "accuracy": 0.77 |
| }, |
| "score": { |
| "correct": 523, |
| "n": 800, |
| "accuracy": 0.65375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.6625, |
| 0.7035 |
| ], |
| "mean_prompt_tokens": 296.0125 |
| }, |
| "v3pre-engram_off|typed_es": { |
| "n": 1985, |
| "correct": 1337, |
| "accuracy": 0.673551637279597, |
| "nll": 0.7705571739247007, |
| "kl": 0.18393918968515446, |
| "ce_soft": 0.952208372298875, |
| "brier": 0.10402584901080503, |
| "brier_mean_k": 0.03191187445888587, |
| "brier_onehot": 0.4470568260447634, |
| "ece10": 0.0919507854080921, |
| "mean_conf": 0.5816008518715049, |
| "by_type": { |
| "choice": { |
| "correct": 354, |
| "n": 594, |
| "accuracy": 0.5959595959595959 |
| }, |
| "noul": { |
| "correct": 462, |
| "n": 597, |
| "accuracy": 0.7738693467336684 |
| }, |
| "score": { |
| "correct": 521, |
| "n": 794, |
| "accuracy": 0.6561712846347607 |
| } |
| }, |
| "ci95_cases": [ |
| 0.6518891687657431, |
| 0.6957178841309823 |
| ], |
| "mean_prompt_tokens": 317.00705289672544 |
| } |
| }, |
| "paired": { |
| "v2 vs v3|typed_en": { |
| "n_common": 2000, |
| "only_b_right": 115, |
| "only_a_right": 97, |
| "p_mcnemar_exact": 0.24291603773847753 |
| }, |
| "v2 vs v3pre|typed_en": { |
| "n_common": 2000, |
| "only_b_right": 145, |
| "only_a_right": 204, |
| "p_mcnemar_exact": 0.001863685262371351 |
| }, |
| "v3 vs v3pre|typed_en": { |
| "n_common": 2000, |
| "only_b_right": 139, |
| "only_a_right": 216, |
| "p_mcnemar_exact": 5.157665267658736e-05 |
| }, |
| "v2 vs v2-engram_off|typed_en": { |
| "n_common": 2000, |
| "only_b_right": 87, |
| "only_a_right": 118, |
| "p_mcnemar_exact": 0.03588773130337706 |
| }, |
| "v3 vs v3-engram_off|typed_en": { |
| "n_common": 2000, |
| "only_b_right": 80, |
| "only_a_right": 114, |
| "p_mcnemar_exact": 0.017596428848766322 |
| }, |
| "v3pre vs v3pre-engram_off|typed_en": { |
| "n_common": 2000, |
| "only_b_right": 84, |
| "only_a_right": 169, |
| "p_mcnemar_exact": 9.833042945424506e-08 |
| }, |
| "v2 vs v3|typed_es": { |
| "n_common": 1985, |
| "only_b_right": 93, |
| "only_a_right": 79, |
| "p_mcnemar_exact": 0.3215739368880947 |
| }, |
| "v2 vs v3pre|typed_es": { |
| "n_common": 1985, |
| "only_b_right": 140, |
| "only_a_right": 225, |
| "p_mcnemar_exact": 1.0076541881668355e-05 |
| }, |
| "v3 vs v3pre|typed_es": { |
| "n_common": 1985, |
| "only_b_right": 124, |
| "only_a_right": 223, |
| "p_mcnemar_exact": 1.186049100732044e-07 |
| }, |
| "v2 vs v2-engram_off|typed_es": { |
| "n_common": 1985, |
| "only_b_right": 84, |
| "only_a_right": 139, |
| "p_mcnemar_exact": 0.00028038310851922213 |
| }, |
| "v3 vs v3-engram_off|typed_es": { |
| "n_common": 1985, |
| "only_b_right": 86, |
| "only_a_right": 136, |
| "p_mcnemar_exact": 0.000963416059185833 |
| }, |
| "v3pre vs v3pre-engram_off|typed_es": { |
| "n_common": 1985, |
| "only_b_right": 112, |
| "only_a_right": 172, |
| "p_mcnemar_exact": 0.00044348896426782365 |
| }, |
| "v2 vs v3|telepatia_es": { |
| "n_common": 2000, |
| "only_b_right": 123, |
| "only_a_right": 89, |
| "p_mcnemar_exact": 0.023197648128185373 |
| }, |
| "v2 vs v3pre|telepatia_es": { |
| "n_common": 2000, |
| "only_b_right": 142, |
| "only_a_right": 211, |
| "p_mcnemar_exact": 0.00028369251691473394 |
| }, |
| "v3 vs v3pre|telepatia_es": { |
| "n_common": 2000, |
| "only_b_right": 127, |
| "only_a_right": 230, |
| "p_mcnemar_exact": 5.4806422818573034e-08 |
| }, |
| "v2 vs v2-engram_off|telepatia_es": { |
| "n_common": 2000, |
| "only_b_right": 73, |
| "only_a_right": 132, |
| "p_mcnemar_exact": 4.561358490864986e-05 |
| }, |
| "v3 vs v3-engram_off|telepatia_es": { |
| "n_common": 2000, |
| "only_b_right": 92, |
| "only_a_right": 149, |
| "p_mcnemar_exact": 0.00029170212350159255 |
| }, |
| "v3pre vs v3pre-engram_off|telepatia_es": { |
| "n_common": 2000, |
| "only_b_right": 102, |
| "only_a_right": 156, |
| "p_mcnemar_exact": 0.0009318393275174185 |
| }, |
| "v2 vs v3|tasksource": { |
| "n_common": 2206, |
| "only_b_right": 619, |
| "only_a_right": 376, |
| "p_mcnemar_exact": 1.2590765149138642e-14 |
| }, |
| "v2 vs v3pre|tasksource": { |
| "n_common": 2206, |
| "only_b_right": 651, |
| "only_a_right": 383, |
| "p_mcnemar_exact": 6.8319004631405e-17 |
| }, |
| "v3 vs v3pre|tasksource": { |
| "n_common": 2206, |
| "only_b_right": 238, |
| "only_a_right": 213, |
| "p_mcnemar_exact": 0.25840324588421176 |
| }, |
| "v2 vs v2-engram_off|tasksource": { |
| "n_common": 2206, |
| "only_b_right": 117, |
| "only_a_right": 151, |
| "p_mcnemar_exact": 0.04361997977786522 |
| }, |
| "v3 vs v3-engram_off|tasksource": { |
| "n_common": 2206, |
| "only_b_right": 178, |
| "only_a_right": 234, |
| "p_mcnemar_exact": 0.00666462862779683 |
| }, |
| "v3pre vs v3pre-engram_off|tasksource": { |
| "n_common": 2206, |
| "only_b_right": 187, |
| "only_a_right": 222, |
| "p_mcnemar_exact": 0.09260412374435281 |
| } |
| }, |
| "typed_en_by_teacher_agreement": { |
| "v2": { |
| "split": { |
| "n": 812, |
| "accuracy": 0.6059113300492611 |
| }, |
| "agreed": { |
| "n": 1188, |
| "accuracy": 0.8569023569023569 |
| } |
| }, |
| "v2-engram_off": { |
| "split": { |
| "n": 812, |
| "accuracy": 0.6009852216748769 |
| }, |
| "agreed": { |
| "n": 1188, |
| "accuracy": 0.8341750841750841 |
| } |
| }, |
| "v3": { |
| "split": { |
| "n": 812, |
| "accuracy": 0.6169950738916257 |
| }, |
| "agreed": { |
| "n": 1188, |
| "accuracy": 0.8644781144781145 |
| } |
| }, |
| "v3-engram_off": { |
| "split": { |
| "n": 812, |
| "accuracy": 0.6009852216748769 |
| }, |
| "agreed": { |
| "n": 1188, |
| "accuracy": 0.8468013468013468 |
| } |
| }, |
| "v3pre": { |
| "split": { |
| "n": 812, |
| "accuracy": 0.5874384236453202 |
| }, |
| "agreed": { |
| "n": 1188, |
| "accuracy": 0.8198653198653199 |
| } |
| }, |
| "v3pre-engram_off": { |
| "split": { |
| "n": 812, |
| "accuracy": 0.5455665024630542 |
| }, |
| "agreed": { |
| "n": 1188, |
| "accuracy": 0.7769360269360269 |
| } |
| } |
| }, |
| "tasksource": { |
| "v2": { |
| "by_family": { |
| "CONDAQA": [ |
| 3, |
| 10 |
| ], |
| "CREAK": [ |
| 5, |
| 10 |
| ], |
| "FLUTE": [ |
| 6, |
| 10 |
| ], |
| "FOL-nli": [ |
| 3, |
| 10 |
| ], |
| "HatemojiBuild": [ |
| 3, |
| 10 |
| ], |
| "HelpSteer": [ |
| 4, |
| 14 |
| ], |
| "HelpSteer2": [ |
| 8, |
| 26 |
| ], |
| "HelpSteer3": [ |
| 15, |
| 48 |
| ], |
| "I2D2": [ |
| 4, |
| 10 |
| ], |
| "MSciNLI": [ |
| 4, |
| 10 |
| ], |
| "MedQA-USMLE-4-options-hf": [ |
| 1, |
| 10 |
| ], |
| "PARADISE": [ |
| 5, |
| 10 |
| ], |
| "PARARULE-Plus": [ |
| 6, |
| 10 |
| ], |
| "Prompt-injection-dataset": [ |
| 5, |
| 10 |
| ], |
| "PromptShield": [ |
| 4, |
| 10 |
| ], |
| "SDOH-NLI": [ |
| 8, |
| 10 |
| ], |
| "SpaceNLI": [ |
| 3, |
| 10 |
| ], |
| "Touche23-ValueEval": [ |
| 7, |
| 10 |
| ], |
| "TroFi": [ |
| 4, |
| 10 |
| ], |
| "TuringBench": [ |
| 2, |
| 10 |
| ], |
| "UNLI": [ |
| 3, |
| 10 |
| ], |
| "UltraFeedback-paired": [ |
| 7, |
| 10 |
| ], |
| "WANLI": [ |
| 5, |
| 10 |
| ], |
| "acceptability-prediction": [ |
| 7, |
| 20 |
| ], |
| "agent_action_safety": [ |
| 6, |
| 10 |
| ], |
| "ai2_arc": [ |
| 6, |
| 20 |
| ], |
| "amazon_counterfactual": [ |
| 4, |
| 10 |
| ], |
| "amazon_polarity": [ |
| 5, |
| 10 |
| ], |
| "arct": [ |
| 5, |
| 10 |
| ], |
| "args_me": [ |
| 6, |
| 10 |
| ], |
| "autotnli": [ |
| 5, |
| 10 |
| ], |
| "babi_nli": [ |
| 17, |
| 35 |
| ], |
| "balanced-copa": [ |
| 5, |
| 10 |
| ], |
| "banking77": [ |
| 1, |
| 2 |
| ], |
| "blog_authorship_corpus": [ |
| 7, |
| 18 |
| ], |
| "breaking_nli": [ |
| 2, |
| 10 |
| ], |
| "cicero": [ |
| 2, |
| 10 |
| ], |
| "circa": [ |
| 2, |
| 10 |
| ], |
| "civil_comments": [ |
| 60, |
| 64 |
| ], |
| "cladder": [ |
| 5, |
| 10 |
| ], |
| "clcd-english": [ |
| 5, |
| 10 |
| ], |
| "cloth": [ |
| 1, |
| 10 |
| ], |
| "codah": [ |
| 2, |
| 10 |
| ], |
| "commonsense_qa": [ |
| 3, |
| 10 |
| ], |
| "commonsense_qa_2.0": [ |
| 4, |
| 10 |
| ], |
| "conceptrules_v2": [ |
| 6, |
| 10 |
| ], |
| "corr2cause": [ |
| 3, |
| 10 |
| ], |
| "cosmos_qa": [ |
| 2, |
| 10 |
| ], |
| "cycic_classification": [ |
| 2, |
| 10 |
| ], |
| "cycic_multiplechoice": [ |
| 3, |
| 10 |
| ], |
| "dbpedia_14": [ |
| 0, |
| 10 |
| ], |
| "defeasible-nli": [ |
| 13, |
| 22 |
| ], |
| "dgen": [ |
| 2, |
| 10 |
| ], |
| "discovery": [ |
| 0, |
| 2 |
| ], |
| "disrpt": [ |
| 2, |
| 10 |
| ], |
| "dnd_style_intents": [ |
| 2, |
| 10 |
| ], |
| "doc-nli": [ |
| 5, |
| 10 |
| ], |
| "dynahate": [ |
| 5, |
| 10 |
| ], |
| "dynasent": [ |
| 6, |
| 20 |
| ], |
| "e-CARE": [ |
| 6, |
| 10 |
| ], |
| "emo": [ |
| 1, |
| 10 |
| ], |
| "equate": [ |
| 5, |
| 10 |
| ], |
| "esci": [ |
| 4, |
| 10 |
| ], |
| "ethics": [ |
| 12, |
| 22 |
| ], |
| "ethos": [ |
| 2, |
| 10 |
| ], |
| "feasibilityQA": [ |
| 2, |
| 10 |
| ], |
| "fever-evidence-related": [ |
| 7, |
| 10 |
| ], |
| "fig-qa": [ |
| 4, |
| 10 |
| ], |
| "folio": [ |
| 3, |
| 10 |
| ], |
| "fool-me-twice": [ |
| 7, |
| 10 |
| ], |
| "gen_debiased_nli": [ |
| 0, |
| 2 |
| ], |
| "github-issue-similarity": [ |
| 7, |
| 10 |
| ], |
| "go_emotions": [ |
| 0, |
| 1 |
| ], |
| "goal-step-wikihow": [ |
| 15, |
| 30 |
| ], |
| "google_wellformed_query": [ |
| 7, |
| 10 |
| ], |
| "hate_speech18": [ |
| 4, |
| 10 |
| ], |
| "head_qa": [ |
| 2, |
| 10 |
| ], |
| "headline_cause": [ |
| 2, |
| 9 |
| ], |
| "health_fact": [ |
| 3, |
| 10 |
| ], |
| "help-nli": [ |
| 6, |
| 10 |
| ], |
| "hh-rlhf": [ |
| 16, |
| 40 |
| ], |
| "hlgd": [ |
| 3, |
| 10 |
| ], |
| "hope_edi": [ |
| 9, |
| 10 |
| ], |
| "hover": [ |
| 5, |
| 10 |
| ], |
| "hover-3way": [ |
| 5, |
| 10 |
| ], |
| "hyperpartisan_news": [ |
| 1, |
| 9 |
| ], |
| "implicatures": [ |
| 7, |
| 10 |
| ], |
| "it-support-tickets": [ |
| 1, |
| 10 |
| ], |
| "jigsaw_toxicity": [ |
| 2, |
| 10 |
| ], |
| "lex_glue": [ |
| 8, |
| 30 |
| ], |
| "logical-entailment": [ |
| 6, |
| 10 |
| ], |
| "lonli": [ |
| 6, |
| 10 |
| ], |
| "math_qa": [ |
| 2, |
| 9 |
| ], |
| "medmcqa": [ |
| 3, |
| 10 |
| ], |
| "mindgames": [ |
| 7, |
| 10 |
| ], |
| "monotonicity-entailment": [ |
| 7, |
| 10 |
| ], |
| "moral_stories": [ |
| 4, |
| 10 |
| ], |
| "multilingual": [ |
| 111, |
| 390 |
| ], |
| "nan-nli": [ |
| 5, |
| 10 |
| ], |
| "naturallogic": [ |
| 1, |
| 10 |
| ], |
| "numer_sense": [ |
| 2, |
| 10 |
| ], |
| "onestop_qa": [ |
| 4, |
| 10 |
| ], |
| "open_question_type": [ |
| 1, |
| 10 |
| ], |
| "patent-phrase-similarity": [ |
| 2, |
| 10 |
| ], |
| "poem_sentiment": [ |
| 1, |
| 10 |
| ], |
| "prm800k_dpo": [ |
| 10, |
| 20 |
| ], |
| "probability_words_nli": [ |
| 11, |
| 23 |
| ], |
| "prompt-injection-dataset": [ |
| 7, |
| 10 |
| ], |
| "prompt-injections": [ |
| 3, |
| 10 |
| ], |
| "propsegment": [ |
| 3, |
| 10 |
| ], |
| "prost": [ |
| 3, |
| 10 |
| ], |
| "proto_qa": [ |
| 3, |
| 10 |
| ], |
| "puzzte": [ |
| 6, |
| 10 |
| ], |
| "qasc": [ |
| 1, |
| 10 |
| ], |
| "quartz": [ |
| 5, |
| 10 |
| ], |
| "recast": [ |
| 10, |
| 22 |
| ], |
| "resnli": [ |
| 6, |
| 10 |
| ], |
| "robust_nli": [ |
| 1, |
| 4 |
| ], |
| "robust_nli_is_sd": [ |
| 1, |
| 3 |
| ], |
| "robust_nli_li_ts": [ |
| 1, |
| 2 |
| ], |
| "ruletaker": [ |
| 4, |
| 10 |
| ], |
| "rumoureval_2019": [ |
| 8, |
| 10 |
| ], |
| "scientific-exaggeration-detection": [ |
| 4, |
| 10 |
| ], |
| "scinli": [ |
| 2, |
| 10 |
| ], |
| "scone": [ |
| 3, |
| 10 |
| ], |
| "scruples": [ |
| 3, |
| 10 |
| ], |
| "sem_eval_2010_task_8": [ |
| 4, |
| 10 |
| ], |
| "shell-safety-v2": [ |
| 3, |
| 10 |
| ], |
| "sherliic": [ |
| 5, |
| 10 |
| ], |
| "silicone": [ |
| 0, |
| 8 |
| ], |
| "sms_spam": [ |
| 2, |
| 10 |
| ], |
| "snips_built_in_intents": [ |
| 3, |
| 10 |
| ], |
| "snli": [ |
| 4, |
| 10 |
| ], |
| "social_i_qa": [ |
| 4, |
| 10 |
| ], |
| "spartqa-mchoice": [ |
| 3, |
| 10 |
| ], |
| "spartqa-yn": [ |
| 4, |
| 10 |
| ], |
| "stackoverflow-questions": [ |
| 2, |
| 10 |
| ], |
| "starcon": [ |
| 8, |
| 10 |
| ], |
| "stepgame": [ |
| 2, |
| 10 |
| ], |
| "sts-companion": [ |
| 2, |
| 10 |
| ], |
| "subjectivity": [ |
| 6, |
| 10 |
| ], |
| "syntactic-augmentation-nli": [ |
| 2, |
| 10 |
| ], |
| "temporal-nli": [ |
| 3, |
| 10 |
| ], |
| "tomi-nli": [ |
| 7, |
| 10 |
| ], |
| "tracie": [ |
| 6, |
| 10 |
| ], |
| "tweets_hate_speech_detection": [ |
| 1, |
| 10 |
| ], |
| "twitter-financial-news-sentiment": [ |
| 1, |
| 10 |
| ], |
| "utilitarianism": [ |
| 4, |
| 10 |
| ], |
| "vitaminc": [ |
| 5, |
| 10 |
| ], |
| "wice": [ |
| 1, |
| 10 |
| ], |
| "wiki_hop": [ |
| 1, |
| 10 |
| ], |
| "wikimedqa": [ |
| 2, |
| 10 |
| ], |
| "wildguardmix-cleaned": [ |
| 10, |
| 21 |
| ], |
| "winodict": [ |
| 4, |
| 10 |
| ], |
| "winowhy": [ |
| 6, |
| 10 |
| ], |
| "wnut_17": [ |
| 2, |
| 10 |
| ], |
| "wouldyourather": [ |
| 4, |
| 10 |
| ] |
| }, |
| "by_state_overlap": { |
| "state_text_not_in_train": [ |
| 845, |
| 2155 |
| ], |
| "state_text_in_train": [ |
| 5, |
| 51 |
| ] |
| }, |
| "by_variant": { |
| "direct": [ |
| 692, |
| 1814 |
| ], |
| "packed_derived": [ |
| 66, |
| 164 |
| ], |
| "label_verification": [ |
| 42, |
| 87 |
| ], |
| "instruction_paraphrase": [ |
| 8, |
| 40 |
| ], |
| "criteria_permutation": [ |
| 35, |
| 80 |
| ], |
| "paired_text_format": [ |
| 7, |
| 21 |
| ] |
| } |
| }, |
| "v3": { |
| "by_family": { |
| "CONDAQA": [ |
| 4, |
| 10 |
| ], |
| "CREAK": [ |
| 7, |
| 10 |
| ], |
| "FLUTE": [ |
| 6, |
| 10 |
| ], |
| "FOL-nli": [ |
| 7, |
| 10 |
| ], |
| "HatemojiBuild": [ |
| 5, |
| 10 |
| ], |
| "HelpSteer": [ |
| 6, |
| 14 |
| ], |
| "HelpSteer2": [ |
| 15, |
| 26 |
| ], |
| "HelpSteer3": [ |
| 16, |
| 48 |
| ], |
| "I2D2": [ |
| 6, |
| 10 |
| ], |
| "MSciNLI": [ |
| 1, |
| 10 |
| ], |
| "MedQA-USMLE-4-options-hf": [ |
| 5, |
| 10 |
| ], |
| "PARADISE": [ |
| 8, |
| 10 |
| ], |
| "PARARULE-Plus": [ |
| 7, |
| 10 |
| ], |
| "Prompt-injection-dataset": [ |
| 10, |
| 10 |
| ], |
| "PromptShield": [ |
| 7, |
| 10 |
| ], |
| "SDOH-NLI": [ |
| 9, |
| 10 |
| ], |
| "SpaceNLI": [ |
| 5, |
| 10 |
| ], |
| "Touche23-ValueEval": [ |
| 6, |
| 10 |
| ], |
| "TroFi": [ |
| 4, |
| 10 |
| ], |
| "TuringBench": [ |
| 1, |
| 10 |
| ], |
| "UNLI": [ |
| 6, |
| 10 |
| ], |
| "UltraFeedback-paired": [ |
| 4, |
| 10 |
| ], |
| "WANLI": [ |
| 2, |
| 10 |
| ], |
| "acceptability-prediction": [ |
| 8, |
| 20 |
| ], |
| "agent_action_safety": [ |
| 4, |
| 10 |
| ], |
| "ai2_arc": [ |
| 12, |
| 20 |
| ], |
| "amazon_counterfactual": [ |
| 7, |
| 10 |
| ], |
| "amazon_polarity": [ |
| 7, |
| 10 |
| ], |
| "arct": [ |
| 4, |
| 10 |
| ], |
| "args_me": [ |
| 4, |
| 10 |
| ], |
| "autotnli": [ |
| 5, |
| 10 |
| ], |
| "babi_nli": [ |
| 18, |
| 35 |
| ], |
| "balanced-copa": [ |
| 7, |
| 10 |
| ], |
| "banking77": [ |
| 2, |
| 2 |
| ], |
| "blog_authorship_corpus": [ |
| 6, |
| 18 |
| ], |
| "breaking_nli": [ |
| 7, |
| 10 |
| ], |
| "cicero": [ |
| 7, |
| 10 |
| ], |
| "circa": [ |
| 5, |
| 10 |
| ], |
| "civil_comments": [ |
| 61, |
| 64 |
| ], |
| "cladder": [ |
| 3, |
| 10 |
| ], |
| "clcd-english": [ |
| 4, |
| 10 |
| ], |
| "cloth": [ |
| 4, |
| 10 |
| ], |
| "codah": [ |
| 1, |
| 10 |
| ], |
| "commonsense_qa": [ |
| 1, |
| 10 |
| ], |
| "commonsense_qa_2.0": [ |
| 5, |
| 10 |
| ], |
| "conceptrules_v2": [ |
| 3, |
| 10 |
| ], |
| "corr2cause": [ |
| 7, |
| 10 |
| ], |
| "cosmos_qa": [ |
| 3, |
| 10 |
| ], |
| "cycic_classification": [ |
| 5, |
| 10 |
| ], |
| "cycic_multiplechoice": [ |
| 2, |
| 10 |
| ], |
| "dbpedia_14": [ |
| 9, |
| 10 |
| ], |
| "defeasible-nli": [ |
| 13, |
| 22 |
| ], |
| "dgen": [ |
| 6, |
| 10 |
| ], |
| "discovery": [ |
| 1, |
| 2 |
| ], |
| "disrpt": [ |
| 2, |
| 10 |
| ], |
| "dnd_style_intents": [ |
| 3, |
| 10 |
| ], |
| "doc-nli": [ |
| 5, |
| 10 |
| ], |
| "dynahate": [ |
| 1, |
| 10 |
| ], |
| "dynasent": [ |
| 11, |
| 20 |
| ], |
| "e-CARE": [ |
| 7, |
| 10 |
| ], |
| "emo": [ |
| 7, |
| 10 |
| ], |
| "equate": [ |
| 3, |
| 10 |
| ], |
| "esci": [ |
| 9, |
| 10 |
| ], |
| "ethics": [ |
| 11, |
| 22 |
| ], |
| "ethos": [ |
| 4, |
| 10 |
| ], |
| "feasibilityQA": [ |
| 4, |
| 10 |
| ], |
| "fever-evidence-related": [ |
| 9, |
| 10 |
| ], |
| "fig-qa": [ |
| 5, |
| 10 |
| ], |
| "folio": [ |
| 4, |
| 10 |
| ], |
| "fool-me-twice": [ |
| 7, |
| 10 |
| ], |
| "gen_debiased_nli": [ |
| 2, |
| 2 |
| ], |
| "github-issue-similarity": [ |
| 6, |
| 10 |
| ], |
| "go_emotions": [ |
| 1, |
| 1 |
| ], |
| "goal-step-wikihow": [ |
| 16, |
| 30 |
| ], |
| "google_wellformed_query": [ |
| 7, |
| 10 |
| ], |
| "hate_speech18": [ |
| 10, |
| 10 |
| ], |
| "head_qa": [ |
| 4, |
| 10 |
| ], |
| "headline_cause": [ |
| 7, |
| 9 |
| ], |
| "health_fact": [ |
| 4, |
| 10 |
| ], |
| "help-nli": [ |
| 6, |
| 10 |
| ], |
| "hh-rlhf": [ |
| 29, |
| 40 |
| ], |
| "hlgd": [ |
| 8, |
| 10 |
| ], |
| "hope_edi": [ |
| 8, |
| 10 |
| ], |
| "hover": [ |
| 6, |
| 10 |
| ], |
| "hover-3way": [ |
| 3, |
| 10 |
| ], |
| "hyperpartisan_news": [ |
| 6, |
| 9 |
| ], |
| "implicatures": [ |
| 4, |
| 10 |
| ], |
| "it-support-tickets": [ |
| 5, |
| 10 |
| ], |
| "jigsaw_toxicity": [ |
| 8, |
| 10 |
| ], |
| "lex_glue": [ |
| 17, |
| 30 |
| ], |
| "logical-entailment": [ |
| 6, |
| 10 |
| ], |
| "lonli": [ |
| 1, |
| 10 |
| ], |
| "math_qa": [ |
| 3, |
| 9 |
| ], |
| "medmcqa": [ |
| 1, |
| 10 |
| ], |
| "mindgames": [ |
| 8, |
| 10 |
| ], |
| "monotonicity-entailment": [ |
| 6, |
| 10 |
| ], |
| "moral_stories": [ |
| 6, |
| 10 |
| ], |
| "multilingual": [ |
| 162, |
| 390 |
| ], |
| "nan-nli": [ |
| 6, |
| 10 |
| ], |
| "naturallogic": [ |
| 3, |
| 10 |
| ], |
| "numer_sense": [ |
| 0, |
| 10 |
| ], |
| "onestop_qa": [ |
| 3, |
| 10 |
| ], |
| "open_question_type": [ |
| 7, |
| 10 |
| ], |
| "patent-phrase-similarity": [ |
| 4, |
| 10 |
| ], |
| "poem_sentiment": [ |
| 8, |
| 10 |
| ], |
| "prm800k_dpo": [ |
| 11, |
| 20 |
| ], |
| "probability_words_nli": [ |
| 13, |
| 23 |
| ], |
| "prompt-injection-dataset": [ |
| 5, |
| 10 |
| ], |
| "prompt-injections": [ |
| 4, |
| 10 |
| ], |
| "propsegment": [ |
| 6, |
| 10 |
| ], |
| "prost": [ |
| 4, |
| 10 |
| ], |
| "proto_qa": [ |
| 3, |
| 10 |
| ], |
| "puzzte": [ |
| 4, |
| 10 |
| ], |
| "qasc": [ |
| 6, |
| 10 |
| ], |
| "quartz": [ |
| 6, |
| 10 |
| ], |
| "recast": [ |
| 12, |
| 22 |
| ], |
| "resnli": [ |
| 5, |
| 10 |
| ], |
| "robust_nli": [ |
| 2, |
| 4 |
| ], |
| "robust_nli_is_sd": [ |
| 2, |
| 3 |
| ], |
| "robust_nli_li_ts": [ |
| 1, |
| 2 |
| ], |
| "ruletaker": [ |
| 4, |
| 10 |
| ], |
| "rumoureval_2019": [ |
| 5, |
| 10 |
| ], |
| "scientific-exaggeration-detection": [ |
| 4, |
| 10 |
| ], |
| "scinli": [ |
| 2, |
| 10 |
| ], |
| "scone": [ |
| 4, |
| 10 |
| ], |
| "scruples": [ |
| 4, |
| 10 |
| ], |
| "sem_eval_2010_task_8": [ |
| 1, |
| 10 |
| ], |
| "shell-safety-v2": [ |
| 4, |
| 10 |
| ], |
| "sherliic": [ |
| 6, |
| 10 |
| ], |
| "silicone": [ |
| 1, |
| 8 |
| ], |
| "sms_spam": [ |
| 10, |
| 10 |
| ], |
| "snips_built_in_intents": [ |
| 6, |
| 10 |
| ], |
| "snli": [ |
| 4, |
| 10 |
| ], |
| "social_i_qa": [ |
| 4, |
| 10 |
| ], |
| "spartqa-mchoice": [ |
| 1, |
| 10 |
| ], |
| "spartqa-yn": [ |
| 5, |
| 10 |
| ], |
| "stackoverflow-questions": [ |
| 3, |
| 10 |
| ], |
| "starcon": [ |
| 5, |
| 10 |
| ], |
| "stepgame": [ |
| 1, |
| 10 |
| ], |
| "sts-companion": [ |
| 4, |
| 10 |
| ], |
| "subjectivity": [ |
| 5, |
| 10 |
| ], |
| "syntactic-augmentation-nli": [ |
| 2, |
| 10 |
| ], |
| "temporal-nli": [ |
| 5, |
| 10 |
| ], |
| "tomi-nli": [ |
| 6, |
| 10 |
| ], |
| "tracie": [ |
| 5, |
| 10 |
| ], |
| "tweets_hate_speech_detection": [ |
| 9, |
| 10 |
| ], |
| "twitter-financial-news-sentiment": [ |
| 5, |
| 10 |
| ], |
| "utilitarianism": [ |
| 5, |
| 10 |
| ], |
| "vitaminc": [ |
| 2, |
| 10 |
| ], |
| "wice": [ |
| 7, |
| 10 |
| ], |
| "wiki_hop": [ |
| 1, |
| 10 |
| ], |
| "wikimedqa": [ |
| 2, |
| 10 |
| ], |
| "wildguardmix-cleaned": [ |
| 9, |
| 21 |
| ], |
| "winodict": [ |
| 4, |
| 10 |
| ], |
| "winowhy": [ |
| 4, |
| 10 |
| ], |
| "wnut_17": [ |
| 3, |
| 10 |
| ], |
| "wouldyourather": [ |
| 6, |
| 10 |
| ] |
| }, |
| "by_state_overlap": { |
| "state_text_not_in_train": [ |
| 1080, |
| 2155 |
| ], |
| "state_text_in_train": [ |
| 13, |
| 51 |
| ] |
| }, |
| "by_variant": { |
| "direct": [ |
| 896, |
| 1814 |
| ], |
| "packed_derived": [ |
| 76, |
| 164 |
| ], |
| "label_verification": [ |
| 49, |
| 87 |
| ], |
| "instruction_paraphrase": [ |
| 19, |
| 40 |
| ], |
| "criteria_permutation": [ |
| 42, |
| 80 |
| ], |
| "paired_text_format": [ |
| 11, |
| 21 |
| ] |
| } |
| }, |
| "v3pre": { |
| "by_family": { |
| "CONDAQA": [ |
| 6, |
| 10 |
| ], |
| "CREAK": [ |
| 6, |
| 10 |
| ], |
| "FLUTE": [ |
| 4, |
| 10 |
| ], |
| "FOL-nli": [ |
| 4, |
| 10 |
| ], |
| "HatemojiBuild": [ |
| 6, |
| 10 |
| ], |
| "HelpSteer": [ |
| 6, |
| 14 |
| ], |
| "HelpSteer2": [ |
| 16, |
| 26 |
| ], |
| "HelpSteer3": [ |
| 16, |
| 48 |
| ], |
| "I2D2": [ |
| 4, |
| 10 |
| ], |
| "MSciNLI": [ |
| 3, |
| 10 |
| ], |
| "MedQA-USMLE-4-options-hf": [ |
| 3, |
| 10 |
| ], |
| "PARADISE": [ |
| 7, |
| 10 |
| ], |
| "PARARULE-Plus": [ |
| 5, |
| 10 |
| ], |
| "Prompt-injection-dataset": [ |
| 5, |
| 10 |
| ], |
| "PromptShield": [ |
| 7, |
| 10 |
| ], |
| "SDOH-NLI": [ |
| 9, |
| 10 |
| ], |
| "SpaceNLI": [ |
| 5, |
| 10 |
| ], |
| "Touche23-ValueEval": [ |
| 5, |
| 10 |
| ], |
| "TroFi": [ |
| 4, |
| 10 |
| ], |
| "TuringBench": [ |
| 1, |
| 10 |
| ], |
| "UNLI": [ |
| 5, |
| 10 |
| ], |
| "UltraFeedback-paired": [ |
| 6, |
| 10 |
| ], |
| "WANLI": [ |
| 3, |
| 10 |
| ], |
| "acceptability-prediction": [ |
| 9, |
| 20 |
| ], |
| "agent_action_safety": [ |
| 4, |
| 10 |
| ], |
| "ai2_arc": [ |
| 14, |
| 20 |
| ], |
| "amazon_counterfactual": [ |
| 8, |
| 10 |
| ], |
| "amazon_polarity": [ |
| 6, |
| 10 |
| ], |
| "arct": [ |
| 5, |
| 10 |
| ], |
| "args_me": [ |
| 6, |
| 10 |
| ], |
| "autotnli": [ |
| 6, |
| 10 |
| ], |
| "babi_nli": [ |
| 15, |
| 35 |
| ], |
| "balanced-copa": [ |
| 10, |
| 10 |
| ], |
| "banking77": [ |
| 2, |
| 2 |
| ], |
| "blog_authorship_corpus": [ |
| 9, |
| 18 |
| ], |
| "breaking_nli": [ |
| 9, |
| 10 |
| ], |
| "cicero": [ |
| 7, |
| 10 |
| ], |
| "circa": [ |
| 6, |
| 10 |
| ], |
| "civil_comments": [ |
| 61, |
| 64 |
| ], |
| "cladder": [ |
| 6, |
| 10 |
| ], |
| "clcd-english": [ |
| 6, |
| 10 |
| ], |
| "cloth": [ |
| 2, |
| 10 |
| ], |
| "codah": [ |
| 1, |
| 10 |
| ], |
| "commonsense_qa": [ |
| 1, |
| 10 |
| ], |
| "commonsense_qa_2.0": [ |
| 5, |
| 10 |
| ], |
| "conceptrules_v2": [ |
| 6, |
| 10 |
| ], |
| "corr2cause": [ |
| 7, |
| 10 |
| ], |
| "cosmos_qa": [ |
| 2, |
| 10 |
| ], |
| "cycic_classification": [ |
| 5, |
| 10 |
| ], |
| "cycic_multiplechoice": [ |
| 2, |
| 10 |
| ], |
| "dbpedia_14": [ |
| 8, |
| 10 |
| ], |
| "defeasible-nli": [ |
| 13, |
| 22 |
| ], |
| "dgen": [ |
| 8, |
| 10 |
| ], |
| "discovery": [ |
| 1, |
| 2 |
| ], |
| "disrpt": [ |
| 2, |
| 10 |
| ], |
| "dnd_style_intents": [ |
| 3, |
| 10 |
| ], |
| "doc-nli": [ |
| 6, |
| 10 |
| ], |
| "dynahate": [ |
| 3, |
| 10 |
| ], |
| "dynasent": [ |
| 12, |
| 20 |
| ], |
| "e-CARE": [ |
| 7, |
| 10 |
| ], |
| "emo": [ |
| 6, |
| 10 |
| ], |
| "equate": [ |
| 3, |
| 10 |
| ], |
| "esci": [ |
| 9, |
| 10 |
| ], |
| "ethics": [ |
| 11, |
| 22 |
| ], |
| "ethos": [ |
| 8, |
| 10 |
| ], |
| "feasibilityQA": [ |
| 9, |
| 10 |
| ], |
| "fever-evidence-related": [ |
| 7, |
| 10 |
| ], |
| "fig-qa": [ |
| 6, |
| 10 |
| ], |
| "folio": [ |
| 2, |
| 10 |
| ], |
| "fool-me-twice": [ |
| 7, |
| 10 |
| ], |
| "gen_debiased_nli": [ |
| 1, |
| 2 |
| ], |
| "github-issue-similarity": [ |
| 7, |
| 10 |
| ], |
| "go_emotions": [ |
| 1, |
| 1 |
| ], |
| "goal-step-wikihow": [ |
| 15, |
| 30 |
| ], |
| "google_wellformed_query": [ |
| 4, |
| 10 |
| ], |
| "hate_speech18": [ |
| 10, |
| 10 |
| ], |
| "head_qa": [ |
| 3, |
| 10 |
| ], |
| "headline_cause": [ |
| 7, |
| 9 |
| ], |
| "health_fact": [ |
| 4, |
| 10 |
| ], |
| "help-nli": [ |
| 5, |
| 10 |
| ], |
| "hh-rlhf": [ |
| 28, |
| 40 |
| ], |
| "hlgd": [ |
| 8, |
| 10 |
| ], |
| "hope_edi": [ |
| 9, |
| 10 |
| ], |
| "hover": [ |
| 6, |
| 10 |
| ], |
| "hover-3way": [ |
| 2, |
| 10 |
| ], |
| "hyperpartisan_news": [ |
| 4, |
| 9 |
| ], |
| "implicatures": [ |
| 4, |
| 10 |
| ], |
| "it-support-tickets": [ |
| 5, |
| 10 |
| ], |
| "jigsaw_toxicity": [ |
| 8, |
| 10 |
| ], |
| "lex_glue": [ |
| 20, |
| 30 |
| ], |
| "logical-entailment": [ |
| 7, |
| 10 |
| ], |
| "lonli": [ |
| 2, |
| 10 |
| ], |
| "math_qa": [ |
| 3, |
| 9 |
| ], |
| "medmcqa": [ |
| 2, |
| 10 |
| ], |
| "mindgames": [ |
| 9, |
| 10 |
| ], |
| "monotonicity-entailment": [ |
| 5, |
| 10 |
| ], |
| "moral_stories": [ |
| 6, |
| 10 |
| ], |
| "multilingual": [ |
| 169, |
| 390 |
| ], |
| "nan-nli": [ |
| 3, |
| 10 |
| ], |
| "naturallogic": [ |
| 2, |
| 10 |
| ], |
| "numer_sense": [ |
| 1, |
| 10 |
| ], |
| "onestop_qa": [ |
| 3, |
| 10 |
| ], |
| "open_question_type": [ |
| 4, |
| 10 |
| ], |
| "patent-phrase-similarity": [ |
| 3, |
| 10 |
| ], |
| "poem_sentiment": [ |
| 10, |
| 10 |
| ], |
| "prm800k_dpo": [ |
| 11, |
| 20 |
| ], |
| "probability_words_nli": [ |
| 13, |
| 23 |
| ], |
| "prompt-injection-dataset": [ |
| 5, |
| 10 |
| ], |
| "prompt-injections": [ |
| 5, |
| 10 |
| ], |
| "propsegment": [ |
| 4, |
| 10 |
| ], |
| "prost": [ |
| 4, |
| 10 |
| ], |
| "proto_qa": [ |
| 4, |
| 10 |
| ], |
| "puzzte": [ |
| 5, |
| 10 |
| ], |
| "qasc": [ |
| 7, |
| 10 |
| ], |
| "quartz": [ |
| 7, |
| 10 |
| ], |
| "recast": [ |
| 8, |
| 22 |
| ], |
| "resnli": [ |
| 7, |
| 10 |
| ], |
| "robust_nli": [ |
| 2, |
| 4 |
| ], |
| "robust_nli_is_sd": [ |
| 1, |
| 3 |
| ], |
| "robust_nli_li_ts": [ |
| 1, |
| 2 |
| ], |
| "ruletaker": [ |
| 3, |
| 10 |
| ], |
| "rumoureval_2019": [ |
| 6, |
| 10 |
| ], |
| "scientific-exaggeration-detection": [ |
| 5, |
| 10 |
| ], |
| "scinli": [ |
| 4, |
| 10 |
| ], |
| "scone": [ |
| 4, |
| 10 |
| ], |
| "scruples": [ |
| 4, |
| 10 |
| ], |
| "sem_eval_2010_task_8": [ |
| 2, |
| 10 |
| ], |
| "shell-safety-v2": [ |
| 5, |
| 10 |
| ], |
| "sherliic": [ |
| 6, |
| 10 |
| ], |
| "silicone": [ |
| 0, |
| 8 |
| ], |
| "sms_spam": [ |
| 9, |
| 10 |
| ], |
| "snips_built_in_intents": [ |
| 8, |
| 10 |
| ], |
| "snli": [ |
| 3, |
| 10 |
| ], |
| "social_i_qa": [ |
| 2, |
| 10 |
| ], |
| "spartqa-mchoice": [ |
| 3, |
| 10 |
| ], |
| "spartqa-yn": [ |
| 5, |
| 10 |
| ], |
| "stackoverflow-questions": [ |
| 3, |
| 10 |
| ], |
| "starcon": [ |
| 6, |
| 10 |
| ], |
| "stepgame": [ |
| 2, |
| 10 |
| ], |
| "sts-companion": [ |
| 4, |
| 10 |
| ], |
| "subjectivity": [ |
| 5, |
| 10 |
| ], |
| "syntactic-augmentation-nli": [ |
| 2, |
| 10 |
| ], |
| "temporal-nli": [ |
| 2, |
| 10 |
| ], |
| "tomi-nli": [ |
| 6, |
| 10 |
| ], |
| "tracie": [ |
| 2, |
| 10 |
| ], |
| "tweets_hate_speech_detection": [ |
| 9, |
| 10 |
| ], |
| "twitter-financial-news-sentiment": [ |
| 5, |
| 10 |
| ], |
| "utilitarianism": [ |
| 8, |
| 10 |
| ], |
| "vitaminc": [ |
| 2, |
| 10 |
| ], |
| "wice": [ |
| 7, |
| 10 |
| ], |
| "wiki_hop": [ |
| 1, |
| 10 |
| ], |
| "wikimedqa": [ |
| 3, |
| 10 |
| ], |
| "wildguardmix-cleaned": [ |
| 14, |
| 21 |
| ], |
| "winodict": [ |
| 4, |
| 10 |
| ], |
| "winowhy": [ |
| 4, |
| 10 |
| ], |
| "wnut_17": [ |
| 5, |
| 10 |
| ], |
| "wouldyourather": [ |
| 4, |
| 10 |
| ] |
| }, |
| "by_state_overlap": { |
| "state_text_not_in_train": [ |
| 1102, |
| 2155 |
| ], |
| "state_text_in_train": [ |
| 16, |
| 51 |
| ] |
| }, |
| "by_variant": { |
| "direct": [ |
| 924, |
| 1814 |
| ], |
| "packed_derived": [ |
| 82, |
| 164 |
| ], |
| "label_verification": [ |
| 48, |
| 87 |
| ], |
| "instruction_paraphrase": [ |
| 18, |
| 40 |
| ], |
| "criteria_permutation": [ |
| 35, |
| 80 |
| ], |
| "paired_text_format": [ |
| 11, |
| 21 |
| ] |
| } |
| } |
| } |
| } |
|
|
