{
 "lineage": [
  {
   "stage": "pretrain",
   "run_id": "1p6b-pretrain-v3",
   "parent_run_id": null,
   "parent_weights_sha256": null,
   "weights_sha256": "dd3a794d333863035bbe74547a7a310ead79cdc282120c694c49133262d957f9",
   "global_step_at_save": 152588,
   "tokens_seen": 9960847877,
   "training_recipe_hash": "b4ff73d394fba440",
   "dataset_subset_id": "",
   "code_commit": "e723ba145652a3092aea457e498abd8cca27a6a7",
   "config_seed": 1234,
   "saved_at": "2026-09-24T02:13:41",
   "selected_epoch": null,
   "val_score": null
  },
  {
   "stage": "v2",
   "run_id": "decisions-v2-e8",
   "parent_run_id": "1p6b-pretrain-v3",
   "parent_weights_sha256": "dd3a794d333863035bbe74547a7a310ead79cdc282120c694c49133262d957f9",
   "weights_sha256": "d5c76527a9b03c8315df8af7abb55acbfa9ab811c68f55bbd057bf2b796c8fbf",
   "global_step_at_save": 10120,
   "tokens_seen": 0,
   "training_recipe_hash": "1c87e80c550a9e36",
   "dataset_subset_id": "typed_train_en+typed_train_es+jev_train",
   "code_commit": "5fd2f4180d321e69838dd43b6f1cec6da261064a",
   "config_seed": 1234,
   "saved_at": "2026-09-30T20:09:38",
   "selected_epoch": 7,
   "val_score": 0.6914998056527292,
   "train_files": [
    "typed_train_en.jsonl",
    "typed_train_es.jsonl",
    "jev_train.jsonl"
   ],
   "val_files": [
    "typed_val_en.jsonl",
    "typed_val_es.jsonl",
    "jev_val.jsonl"
   ],
   "epochs_scheduled": 8,
   "lr": 1e-05,
   "accum": 16,
   "aux": 0.01,
   "sft_seed": 7,
   "val_curve": [
    0.3706,
    0.4997,
    0.601,
    0.6721,
    0.6709,
    0.675,
    0.6915,
    0.6901
   ],
   "train_questions_per_epoch": 20233,
   "optimizer_steps_scheduled": 10117,
   "optimizer_steps_to_selected_weights": 8852
  },
  {
   "stage": "v3",
   "run_id": "decisions-v3-mix-from-v2",
   "parent_run_id": "decisions-v2-e8",
   "parent_weights_sha256": "d5c76527a9b03c8315df8af7abb55acbfa9ab811c68f55bbd057bf2b796c8fbf",
   "weights_sha256": "5657d3ee7bf9934a52a34f270aa4ebc78ae69ef788ad49c0b07cbd1fe8ebe8fb",
   "global_step_at_save": 10806,
   "tokens_seen": 0,
   "training_recipe_hash": "e4b355103247ef81",
   "dataset_subset_id": "train_helmo+train_jev+train_n4ze3m+train_tasksource+train_typed_en+train_typed_es+train_typed_es_telepatia",
   "code_commit": "0b5e63e3da05c1a51eb9f0ea1758f8d53d200a80-dirty",
   "config_seed": 1234,
   "saved_at": "2026-10-01T13:45:06",
   "selected_epoch": 2,
   "val_score": 0.6444873254407529,
   "train_files": [
    "train_helmo.jsonl",
    "train_jev.jsonl",
    "train_n4ze3m.jsonl",
    "train_tasksource.jsonl",
    "train_typed_en.jsonl",
    "train_typed_es.jsonl",
    "train_typed_es_telepatia.jsonl"
   ],
   "val_files": [
    "val_helmo.jsonl",
    "val_jev.jsonl",
    "val_n4ze3m.jsonl",
    "val_tasksource.jsonl",
    "val_typed_en.jsonl",
    "val_typed_es.jsonl"
   ],
   "epochs_scheduled": 2,
   "lr": 1e-05,
   "accum": 16,
   "aux": 0.01,
   "sft_seed": 7,
   "val_curve": [
    0.6276,
    0.6445
   ],
   "train_questions_per_epoch": 86436,
   "optimizer_steps_scheduled": 10805,
   "optimizer_steps_to_selected_weights": 10805
  },
  {
   "stage": "v3pre",
   "run_id": "decisions-v3-mix",
   "parent_run_id": "1p6b-pretrain-v3",
   "parent_weights_sha256": "dd3a794d333863035bbe74547a7a310ead79cdc282120c694c49133262d957f9",
   "weights_sha256": "221d0663414de1a754bd28499d355d0e3946836cca772bed8dc069a1fd2b9ce4",
   "global_step_at_save": 16209,
   "tokens_seen": 0,
   "training_recipe_hash": "b5f63c4154bf184e",
   "dataset_subset_id": "train_helmo+train_jev+train_n4ze3m+train_tasksource+train_typed_en+train_typed_es+train_typed_es_telepatia",
   "code_commit": "0b5e63e3da05c1a51eb9f0ea1758f8d53d200a80-dirty",
   "config_seed": 1234,
   "saved_at": "2026-10-01T15:53:18",
   "selected_epoch": 3,
   "val_score": 0.6358126982153586,
   "train_files": [
    "train_helmo.jsonl",
    "train_jev.jsonl",
    "train_n4ze3m.jsonl",
    "train_tasksource.jsonl",
    "train_typed_en.jsonl",
    "train_typed_es.jsonl",
    "train_typed_es_telepatia.jsonl"
   ],
   "val_files": [
    "val_helmo.jsonl",
    "val_jev.jsonl",
    "val_n4ze3m.jsonl",
    "val_tasksource.jsonl",
    "val_typed_en.jsonl",
    "val_typed_es.jsonl"
   ],
   "epochs_scheduled": 3,
   "lr": 1e-05,
   "accum": 16,
   "aux": 0.01,
   "sft_seed": 7,
   "val_curve": [
    0.539,
    0.6185,
    0.6358
   ],
   "train_questions_per_epoch": 86436,
   "optimizer_steps_scheduled": 16207,
   "optimizer_steps_to_selected_weights": 16207
  }
 ],
 "reference_rows": {
  "uniform": {
   "n": 2000,
   "correct": 548,
   "accuracy": 0.274,
   "nll": 1.2118217396490383,
   "kl": 0.44446318402470963,
   "ce_soft": 1.2118217396490383,
   "brier": 0.2381881657480489,
   "brier_mean_k": 0.07191006022457894,
   "brier_onehot": 0.6825,
   "ece10": 0.048499999999999995,
   "mean_conf": 0.3175,
   "by_type": {
    "choice": {
     "correct": 119,
     "n": 600,
     "accuracy": 0.19833333333333333
    },
    "noul": {
     "correct": 305,
     "n": 600,
     "accuracy": 0.5083333333333333
    },
    "score": {
     "correct": 124,
     "n": 800,
     "accuracy": 0.155
    }
   },
   "accuracy_expected_1_over_k": 0.3175
  },
  "prior": {
   "n": 2000,
   "correct": 967,
   "accuracy": 0.4835,
   "nll": 1.0147031719053614,
   "kl": 0.4374539124612297,
   "ce_soft": 1.2048124680855585,
   "brier": 0.19727670747180912,
   "brier_mean_k": 0.0579368310718516,
   "brier_onehot": 0.59002,
   "ece10": 0.038666666666666655,
   "mean_conf": 0.5148333333333334,
   "by_type": {
    "choice": {
     "correct": 251,
     "n": 600,
     "accuracy": 0.41833333333333333
    },
    "noul": {
     "correct": 385,
     "n": 600,
     "accuracy": 0.6416666666666667
    },
    "score": {
     "correct": 331,
     "n": 800,
     "accuracy": 0.41375
    }
   }
  },
  "argmax_agree_counts": {
   "false": 812,
   "true": 1188
  }
 },
 "results": {
  "v2|tasksource": {
   "n": 2206,
   "correct": 850,
   "accuracy": 0.38531278331822305,
   "nll": 1.18249501294161,
   "kl": 1.1583903767910129,
   "ce_soft": 1.1835308450719855,
   "brier": 0.6582691515879918,
   "brier_mean_k": 0.2238124108887532,
   "brier_onehot": 0.6728618433752923,
   "ece10": 0.12647392187378995,
   "mean_conf": 0.511786705192013,
   "by_type": {
    "choice": {
     "correct": 664,
     "n": 1815,
     "accuracy": 0.36584022038567493
    },
    "noul": {
     "correct": 151,
     "n": 261,
     "accuracy": 0.578544061302682
    },
    "score": {
     "correct": 35,
     "n": 130,
     "accuracy": 0.2692307692307692
    }
   },
   "ci95_cases": [
    0.3653671804170444,
    0.4070716228467815
   ],
   "mean_prompt_tokens": 206.149138712602
  },
  "v2|telepatia_es": {
   "n": 2000,
   "correct": 1492,
   "accuracy": 0.746,
   "nll": 0.6768666282900727,
   "kl": 0.137470178826373,
   "ce_soft": 0.9048287344507019,
   "brier": 0.07226937229140096,
   "brier_mean_k": 0.020558110269673965,
   "brier_onehot": 0.3838231707888498,
   "ece10": 0.1303335434347391,
   "mean_conf": 0.6159254869967699,
   "by_type": {
    "choice": {
     "correct": 415,
     "n": 600,
     "accuracy": 0.6916666666666667
    },
    "noul": {
     "correct": 502,
     "n": 600,
     "accuracy": 0.8366666666666667
    },
    "score": {
     "correct": 575,
     "n": 800,
     "accuracy": 0.71875
    }
   },
   "ci95_cases": [
    0.7245,
    0.768
   ],
   "mean_prompt_tokens": 315.093
  },
  "v2|typed_en": {
   "n": 2000,
   "correct": 1510,
   "accuracy": 0.755,
   "nll": 0.6463324192750852,
   "kl": 0.13270933329166798,
   "ce_soft": 0.9000678889159969,
   "brier": 0.07002914815029006,
   "brier_mean_k": 0.02004525517631372,
   "brier_onehot": 0.3676075399939686,
   "ece10": 0.12290431407094005,
   "mean_conf": 0.6332694019079208,
   "by_type": {
    "choice": {
     "correct": 411,
     "n": 600,
     "accuracy": 0.685
    },
    "noul": {
     "correct": 503,
     "n": 600,
     "accuracy": 0.8383333333333334
    },
    "score": {
     "correct": 596,
     "n": 800,
     "accuracy": 0.745
    }
   },
   "ci95_cases": [
    0.7335,
    0.7765
   ],
   "mean_prompt_tokens": 296.0125
  },
  "v2|typed_es": {
   "n": 1985,
   "correct": 1482,
   "accuracy": 0.7465994962216624,
   "nll": 0.6550145421374155,
   "kl": 0.13250811726703088,
   "ce_soft": 0.9007772998807515,
   "brier": 0.07028015084314279,
   "brier_mean_k": 0.01999555773940268,
   "brier_onehot": 0.3732510221786229,
   "ece10": 0.1152207864471887,
   "mean_conf": 0.6313787097744737,
   "by_type": {
    "choice": {
     "correct": 401,
     "n": 594,
     "accuracy": 0.6750841750841751
    },
    "noul": {
     "correct": 497,
     "n": 597,
     "accuracy": 0.8324958123953099
    },
    "score": {
     "correct": 584,
     "n": 794,
     "accuracy": 0.7355163727959698
    }
   },
   "ci95_cases": [
    0.7249370277078085,
    0.7677581863979849
   ],
   "mean_prompt_tokens": 317.00705289672544
  },
  "v2-engram_off|tasksource": {
   "n": 2206,
   "correct": 816,
   "accuracy": 0.36990027198549413,
   "nll": 1.177764537558081,
   "kl": 1.1535940646862644,
   "ce_soft": 1.1787345329672367,
   "brier": 0.6549310146754723,
   "brier_mean_k": 0.22274428248490724,
   "brier_onehot": 0.6696232285301682,
   "ece10": 0.1318699845221782,
   "mean_conf": 0.5017702565076723,
   "by_type": {
    "choice": {
     "correct": 634,
     "n": 1815,
     "accuracy": 0.3493112947658402
    },
    "noul": {
     "correct": 146,
     "n": 261,
     "accuracy": 0.5593869731800766
    },
    "score": {
     "correct": 36,
     "n": 130,
     "accuracy": 0.27692307692307694
    }
   },
   "ci95_cases": [
    0.34950135992747056,
    0.39029918404351766
   ],
   "mean_prompt_tokens": 206.149138712602
  },
  "v2-engram_off|telepatia_es": {
   "n": 2000,
   "correct": 1433,
   "accuracy": 0.7165,
   "nll": 0.7350957449054891,
   "kl": 0.1574048776542151,
   "ce_soft": 0.9247634332785439,
   "brier": 0.08390837700265645,
   "brier_mean_k": 0.024310100717596178,
   "brier_onehot": 0.4186372582045989,
   "ece10": 0.13328185617923735,
   "mean_conf": 0.5832181438207626,
   "by_type": {
    "choice": {
     "correct": 404,
     "n": 600,
     "accuracy": 0.6733333333333333
    },
    "noul": {
     "correct": 494,
     "n": 600,
     "accuracy": 0.8233333333333334
    },
    "score": {
     "correct": 535,
     "n": 800,
     "accuracy": 0.66875
    }
   },
   "ci95_cases": [
    0.694,
    0.7385
   ],
   "mean_prompt_tokens": 315.093
  },
  "v2-engram_off|typed_en": {
   "n": 2000,
   "correct": 1479,
   "accuracy": 0.7395,
   "nll": 0.68511629220165,
   "kl": 0.1402235323363397,
   "ce_soft": 0.9075820879606685,
   "brier": 0.07412362822891774,
   "brier_mean_k": 0.02127965516331085,
   "brier_onehot": 0.38946567577027386,
   "ece10": 0.12908081570267677,
   "mean_conf": 0.6104191842973232,
   "by_type": {
    "choice": {
     "correct": 408,
     "n": 600,
     "accuracy": 0.68
    },
    "noul": {
     "correct": 500,
     "n": 600,
     "accuracy": 0.8333333333333334
    },
    "score": {
     "correct": 571,
     "n": 800,
     "accuracy": 0.71375
    }
   },
   "ci95_cases": [
    0.718,
    0.761
   ],
   "mean_prompt_tokens": 296.0125
  },
  "v2-engram_off|typed_es": {
   "n": 1985,
   "correct": 1427,
   "accuracy": 0.7188916876574307,
   "nll": 0.7129132213174406,
   "kl": 0.1504756989477731,
   "ce_soft": 0.9187448815614937,
   "brier": 0.08014936250258083,
   "brier_mean_k": 0.022948228945889947,
   "brier_onehot": 0.4048794955474447,
   "ece10": 0.12493646382084421,
   "mean_conf": 0.6031942302694249,
   "by_type": {
    "choice": {
     "correct": 385,
     "n": 594,
     "accuracy": 0.6481481481481481
    },
    "noul": {
     "correct": 491,
     "n": 597,
     "accuracy": 0.8224455611390284
    },
    "score": {
     "correct": 551,
     "n": 794,
     "accuracy": 0.6939546599496221
    }
   },
   "ci95_cases": [
    0.6972292191435768,
    0.7405541561712846
   ],
   "mean_prompt_tokens": 317.00705289672544
  },
  "v3|tasksource": {
   "n": 2206,
   "correct": 1093,
   "accuracy": 0.49546690843155033,
   "nll": 1.0302639724427658,
   "kl": 1.0076541905677538,
   "ce_soft": 1.0327946588487265,
   "brier": 0.5674609637187716,
   "brier_mean_k": 0.19068295958341383,
   "brier_onehot": 0.5817144817866338,
   "ece10": 0.07126550326429487,
   "mean_conf": 0.5667324116958452,
   "by_type": {
    "choice": {
     "correct": 870,
     "n": 1815,
     "accuracy": 0.4793388429752066
    },
    "noul": {
     "correct": 172,
     "n": 261,
     "accuracy": 0.6590038314176245
    },
    "score": {
     "correct": 51,
     "n": 130,
     "accuracy": 0.3923076923076923
    }
   },
   "ci95_cases": [
    0.4741613780598368,
    0.5158658204895739
   ],
   "mean_prompt_tokens": 206.149138712602
  },
  "v3|telepatia_es": {
   "n": 2000,
   "correct": 1526,
   "accuracy": 0.763,
   "nll": 0.6385563440375813,
   "kl": 0.11925372995655213,
   "ce_soft": 0.886612285580881,
   "brier": 0.06393994905918735,
   "brier_mean_k": 0.018569114373183912,
   "brier_onehot": 0.36353173992639604,
   "ece10": 0.13380355775356292,
   "mean_conf": 0.6294715915620327,
   "by_type": {
    "choice": {
     "correct": 438,
     "n": 600,
     "accuracy": 0.73
    },
    "noul": {
     "correct": 498,
     "n": 600,
     "accuracy": 0.83
    },
    "score": {
     "correct": 590,
     "n": 800,
     "accuracy": 0.7375
    }
   },
   "ci95_cases": [
    0.742,
    0.785
   ],
   "mean_prompt_tokens": 315.093
  },
  "v3|typed_en": {
   "n": 2000,
   "correct": 1528,
   "accuracy": 0.764,
   "nll": 0.6348000373513565,
   "kl": 0.12133635641347341,
   "ce_soft": 0.8886949120378023,
   "brier": 0.0651098222513845,
   "brier_mean_k": 0.019005062125487936,
   "brier_onehot": 0.3611829084554942,
   "ece10": 0.13159668615460393,
   "mean_conf": 0.632403313845396,
   "by_type": {
    "choice": {
     "correct": 429,
     "n": 600,
     "accuracy": 0.715
    },
    "noul": {
     "correct": 503,
     "n": 600,
     "accuracy": 0.8383333333333334
    },
    "score": {
     "correct": 596,
     "n": 800,
     "accuracy": 0.745
    }
   },
   "ci95_cases": [
    0.742,
    0.7855
   ],
   "mean_prompt_tokens": 296.0125
  },
  "v3|typed_es": {
   "n": 1985,
   "correct": 1496,
   "accuracy": 0.7536523929471033,
   "nll": 0.6394842117466736,
   "kl": 0.11898906052638147,
   "ce_soft": 0.887258243140102,
   "brier": 0.06475655764622247,
   "brier_mean_k": 0.018860197074895686,
   "brier_onehot": 0.36670251099112927,
   "ece10": 0.12293764144107136,
   "mean_conf": 0.6313106105219507,
   "by_type": {
    "choice": {
     "correct": 415,
     "n": 594,
     "accuracy": 0.6986531986531986
    },
    "noul": {
     "correct": 494,
     "n": 597,
     "accuracy": 0.8274706867671692
    },
    "score": {
     "correct": 587,
     "n": 794,
     "accuracy": 0.739294710327456
    }
   },
   "ci95_cases": [
    0.7324937027707809,
    0.7743073047858942
   ],
   "mean_prompt_tokens": 317.00705289672544
  },
  "v3-engram_off|tasksource": {
   "n": 2206,
   "correct": 1037,
   "accuracy": 0.4700815956482321,
   "nll": 1.040257259141714,
   "kl": 1.0165084501963875,
   "ce_soft": 1.0416489184773599,
   "brier": 0.5747761215316167,
   "brier_mean_k": 0.19368823630368673,
   "brier_onehot": 0.5896047821488928,
   "ece10": 0.06856633263791244,
   "mean_conf": 0.5386479282861445,
   "by_type": {
    "choice": {
     "correct": 829,
     "n": 1815,
     "accuracy": 0.4567493112947658
    },
    "noul": {
     "correct": 156,
     "n": 261,
     "accuracy": 0.5977011494252874
    },
    "score": {
     "correct": 52,
     "n": 130,
     "accuracy": 0.4
    }
   },
   "ci95_cases": [
    0.4492293744333635,
    0.4904805077062557
   ],
   "mean_prompt_tokens": 206.149138712602
  },
  "v3-engram_off|telepatia_es": {
   "n": 2000,
   "correct": 1469,
   "accuracy": 0.7345,
   "nll": 0.6917027070297807,
   "kl": 0.13656979911128891,
   "ce_soft": 0.9039283547356177,
   "brier": 0.07339688864379973,
   "brier_mean_k": 0.021115876966946143,
   "brier_onehot": 0.3936445649003366,
   "ece10": 0.13314792701601982,
   "mean_conf": 0.6027737509608269,
   "by_type": {
    "choice": {
     "correct": 401,
     "n": 600,
     "accuracy": 0.6683333333333333
    },
    "noul": {
     "correct": 498,
     "n": 600,
     "accuracy": 0.83
    },
    "score": {
     "correct": 570,
     "n": 800,
     "accuracy": 0.7125
    }
   },
   "ci95_cases": [
    0.713,
    0.756
   ],
   "mean_prompt_tokens": 315.093
  },
  "v3-engram_off|typed_en": {
   "n": 2000,
   "correct": 1494,
   "accuracy": 0.747,
   "nll": 0.6658806168427456,
   "kl": 0.1298324631061122,
   "ce_soft": 0.897191018730441,
   "brier": 0.06997183069647757,
   "brier_mean_k": 0.0204340936628309,
   "brier_onehot": 0.3804533671105715,
   "ece10": 0.13033386017382145,
   "mean_conf": 0.6169207523316145,
   "by_type": {
    "choice": {
     "correct": 410,
     "n": 600,
     "accuracy": 0.6833333333333333
    },
    "noul": {
     "correct": 491,
     "n": 600,
     "accuracy": 0.8183333333333334
    },
    "score": {
     "correct": 593,
     "n": 800,
     "accuracy": 0.74125
    }
   },
   "ci95_cases": [
    0.726,
    0.768
   ],
   "mean_prompt_tokens": 296.0125
  },
  "v3-engram_off|typed_es": {
   "n": 1985,
   "correct": 1446,
   "accuracy": 0.728463476070529,
   "nll": 0.6957014567741066,
   "kl": 0.13709141764197266,
   "ce_soft": 0.9053606002556932,
   "brier": 0.07422417302252052,
   "brier_mean_k": 0.021329935813320985,
   "brier_onehot": 0.3961568726644083,
   "ece10": 0.12043466251202734,
   "mean_conf": 0.6082240724293351,
   "by_type": {
    "choice": {
     "correct": 394,
     "n": 594,
     "accuracy": 0.6632996632996633
    },
    "noul": {
     "correct": 492,
     "n": 597,
     "accuracy": 0.8241206030150754
    },
    "score": {
     "correct": 560,
     "n": 794,
     "accuracy": 0.7052896725440806
    }
   },
   "ci95_cases": [
    0.7062972292191436,
    0.7496221662468514
   ],
   "mean_prompt_tokens": 317.00705289672544
  },
  "v3pre|tasksource": {
   "n": 2206,
   "correct": 1118,
   "accuracy": 0.5067996373526745,
   "nll": 1.0126227101604877,
   "kl": 0.989910359101978,
   "ce_soft": 1.0150508273829506,
   "brier": 0.5577300772285442,
   "brier_mean_k": 0.18832110523426265,
   "brier_onehot": 0.5721825731599312,
   "ece10": 0.06798556558483856,
   "mean_conf": 0.5731623702136258,
   "by_type": {
    "choice": {
     "correct": 893,
     "n": 1815,
     "accuracy": 0.4920110192837466
    },
    "noul": {
     "correct": 167,
     "n": 261,
     "accuracy": 0.6398467432950191
    },
    "score": {
     "correct": 58,
     "n": 130,
     "accuracy": 0.4461538461538462
    }
   },
   "ci95_cases": [
    0.485494106980961,
    0.5271985494106981
   ],
   "mean_prompt_tokens": 206.149138712602
  },
  "v3pre|telepatia_es": {
   "n": 2000,
   "correct": 1423,
   "accuracy": 0.7115,
   "nll": 0.7221613642472114,
   "kl": 0.16424527027114952,
   "ce_soft": 0.9316038258954784,
   "brier": 0.09163500709581687,
   "brier_mean_k": 0.027993557290275604,
   "brier_onehot": 0.4151352846821021,
   "ece10": 0.10952522042393688,
   "mean_conf": 0.6024507610797882,
   "by_type": {
    "choice": {
     "correct": 381,
     "n": 600,
     "accuracy": 0.635
    },
    "noul": {
     "correct": 473,
     "n": 600,
     "accuracy": 0.7883333333333333
    },
    "score": {
     "correct": 569,
     "n": 800,
     "accuracy": 0.71125
    }
   },
   "ci95_cases": [
    0.69,
    0.7315
   ],
   "mean_prompt_tokens": 315.093
  },
  "v3pre|typed_en": {
   "n": 2000,
   "correct": 1451,
   "accuracy": 0.7255,
   "nll": 0.7228620920358019,
   "kl": 0.15801520740611583,
   "ce_soft": 0.9253737630304447,
   "brier": 0.08732942362644587,
   "brier_mean_k": 0.026559532585552777,
   "brier_onehot": 0.41381279256051157,
   "ece10": 0.13104377594590189,
   "mean_conf": 0.5947328799664974,
   "by_type": {
    "choice": {
     "correct": 393,
     "n": 600,
     "accuracy": 0.655
    },
    "noul": {
     "correct": 487,
     "n": 600,
     "accuracy": 0.8116666666666666
    },
    "score": {
     "correct": 571,
     "n": 800,
     "accuracy": 0.71375
    }
   },
   "ci95_cases": [
    0.7045,
    0.7455
   ],
   "mean_prompt_tokens": 296.0125
  },
  "v3pre|typed_es": {
   "n": 1985,
   "correct": 1397,
   "accuracy": 0.7037783375314861,
   "nll": 0.7238645309775039,
   "kl": 0.16208690588450078,
   "ce_soft": 0.9303560884982214,
   "brier": 0.09048640592028294,
   "brier_mean_k": 0.027522013017270756,
   "brier_onehot": 0.41629011479103895,
   "ece10": 0.10134872849702231,
   "mean_conf": 0.6027212248340962,
   "by_type": {
    "choice": {
     "correct": 365,
     "n": 594,
     "accuracy": 0.6144781144781145
    },
    "noul": {
     "correct": 475,
     "n": 597,
     "accuracy": 0.7956448911222781
    },
    "score": {
     "correct": 557,
     "n": 794,
     "accuracy": 0.7015113350125944
    }
   },
   "ci95_cases": [
    0.6821158690176322,
    0.7249370277078085
   ],
   "mean_prompt_tokens": 317.00705289672544
  },
  "v3pre-engram_off|tasksource": {
   "n": 2206,
   "correct": 1083,
   "accuracy": 0.4909338168631006,
   "nll": 1.0360851266471505,
   "kl": 1.0127516723754644,
   "ce_soft": 1.037892140656437,
   "brier": 0.5716110348043713,
   "brier_mean_k": 0.1933716392928341,
   "brier_onehot": 0.5863590220270787,
   "ece10": 0.07097867969170227,
   "mean_conf": 0.5581544994504097,
   "by_type": {
    "choice": {
     "correct": 858,
     "n": 1815,
     "accuracy": 0.4727272727272727
    },
    "noul": {
     "correct": 172,
     "n": 261,
     "accuracy": 0.6590038314176245
    },
    "score": {
     "correct": 53,
     "n": 130,
     "accuracy": 0.4076923076923077
    }
   },
   "ci95_cases": [
    0.4700815956482321,
    0.5117860380779692
   ],
   "mean_prompt_tokens": 206.149138712602
  },
  "v3pre-engram_off|telepatia_es": {
   "n": 2000,
   "correct": 1369,
   "accuracy": 0.6845,
   "nll": 0.7664083850685155,
   "kl": 0.1835608149642982,
   "ce_soft": 0.9509193705886271,
   "brier": 0.1026449902821665,
   "brier_mean_k": 0.0314341176817405,
   "brier_onehot": 0.44305319561626966,
   "ece10": 0.1018519453555346,
   "mean_conf": 0.5826480546444655,
   "by_type": {
    "choice": {
     "correct": 376,
     "n": 600,
     "accuracy": 0.6266666666666667
    },
    "noul": {
     "correct": 462,
     "n": 600,
     "accuracy": 0.77
    },
    "score": {
     "correct": 531,
     "n": 800,
     "accuracy": 0.66375
    }
   },
   "ci95_cases": [
    0.6635,
    0.706
   ],
   "mean_prompt_tokens": 315.093
  },
  "v3pre-engram_off|typed_en": {
   "n": 2000,
   "correct": 1366,
   "accuracy": 0.683,
   "nll": 0.7700775281040052,
   "kl": 0.18422638726523183,
   "ce_soft": 0.9515849428895607,
   "brier": 0.10380722453958674,
   "brier_mean_k": 0.03158302297002527,
   "brier_onehot": 0.44732844112781456,
   "ece10": 0.1008905218988657,
   "mean_conf": 0.582200200304389,
   "by_type": {
    "choice": {
     "correct": 381,
     "n": 600,
     "accuracy": 0.635
    },
    "noul": {
     "correct": 462,
     "n": 600,
     "accuracy": 0.77
    },
    "score": {
     "correct": 523,
     "n": 800,
     "accuracy": 0.65375
    }
   },
   "ci95_cases": [
    0.6625,
    0.7035
   ],
   "mean_prompt_tokens": 296.0125
  },
  "v3pre-engram_off|typed_es": {
   "n": 1985,
   "correct": 1337,
   "accuracy": 0.673551637279597,
   "nll": 0.7705571739247007,
   "kl": 0.18393918968515446,
   "ce_soft": 0.952208372298875,
   "brier": 0.10402584901080503,
   "brier_mean_k": 0.03191187445888587,
   "brier_onehot": 0.4470568260447634,
   "ece10": 0.0919507854080921,
   "mean_conf": 0.5816008518715049,
   "by_type": {
    "choice": {
     "correct": 354,
     "n": 594,
     "accuracy": 0.5959595959595959
    },
    "noul": {
     "correct": 462,
     "n": 597,
     "accuracy": 0.7738693467336684
    },
    "score": {
     "correct": 521,
     "n": 794,
     "accuracy": 0.6561712846347607
    }
   },
   "ci95_cases": [
    0.6518891687657431,
    0.6957178841309823
   ],
   "mean_prompt_tokens": 317.00705289672544
  }
 },
 "paired": {
  "v2 vs v3|typed_en": {
   "n_common": 2000,
   "only_b_right": 115,
   "only_a_right": 97,
   "p_mcnemar_exact": 0.24291603773847753
  },
  "v2 vs v3pre|typed_en": {
   "n_common": 2000,
   "only_b_right": 145,
   "only_a_right": 204,
   "p_mcnemar_exact": 0.001863685262371351
  },
  "v3 vs v3pre|typed_en": {
   "n_common": 2000,
   "only_b_right": 139,
   "only_a_right": 216,
   "p_mcnemar_exact": 5.157665267658736e-05
  },
  "v2 vs v2-engram_off|typed_en": {
   "n_common": 2000,
   "only_b_right": 87,
   "only_a_right": 118,
   "p_mcnemar_exact": 0.03588773130337706
  },
  "v3 vs v3-engram_off|typed_en": {
   "n_common": 2000,
   "only_b_right": 80,
   "only_a_right": 114,
   "p_mcnemar_exact": 0.017596428848766322
  },
  "v3pre vs v3pre-engram_off|typed_en": {
   "n_common": 2000,
   "only_b_right": 84,
   "only_a_right": 169,
   "p_mcnemar_exact": 9.833042945424506e-08
  },
  "v2 vs v3|typed_es": {
   "n_common": 1985,
   "only_b_right": 93,
   "only_a_right": 79,
   "p_mcnemar_exact": 0.3215739368880947
  },
  "v2 vs v3pre|typed_es": {
   "n_common": 1985,
   "only_b_right": 140,
   "only_a_right": 225,
   "p_mcnemar_exact": 1.0076541881668355e-05
  },
  "v3 vs v3pre|typed_es": {
   "n_common": 1985,
   "only_b_right": 124,
   "only_a_right": 223,
   "p_mcnemar_exact": 1.186049100732044e-07
  },
  "v2 vs v2-engram_off|typed_es": {
   "n_common": 1985,
   "only_b_right": 84,
   "only_a_right": 139,
   "p_mcnemar_exact": 0.00028038310851922213
  },
  "v3 vs v3-engram_off|typed_es": {
   "n_common": 1985,
   "only_b_right": 86,
   "only_a_right": 136,
   "p_mcnemar_exact": 0.000963416059185833
  },
  "v3pre vs v3pre-engram_off|typed_es": {
   "n_common": 1985,
   "only_b_right": 112,
   "only_a_right": 172,
   "p_mcnemar_exact": 0.00044348896426782365
  },
  "v2 vs v3|telepatia_es": {
   "n_common": 2000,
   "only_b_right": 123,
   "only_a_right": 89,
   "p_mcnemar_exact": 0.023197648128185373
  },
  "v2 vs v3pre|telepatia_es": {
   "n_common": 2000,
   "only_b_right": 142,
   "only_a_right": 211,
   "p_mcnemar_exact": 0.00028369251691473394
  },
  "v3 vs v3pre|telepatia_es": {
   "n_common": 2000,
   "only_b_right": 127,
   "only_a_right": 230,
   "p_mcnemar_exact": 5.4806422818573034e-08
  },
  "v2 vs v2-engram_off|telepatia_es": {
   "n_common": 2000,
   "only_b_right": 73,
   "only_a_right": 132,
   "p_mcnemar_exact": 4.561358490864986e-05
  },
  "v3 vs v3-engram_off|telepatia_es": {
   "n_common": 2000,
   "only_b_right": 92,
   "only_a_right": 149,
   "p_mcnemar_exact": 0.00029170212350159255
  },
  "v3pre vs v3pre-engram_off|telepatia_es": {
   "n_common": 2000,
   "only_b_right": 102,
   "only_a_right": 156,
   "p_mcnemar_exact": 0.0009318393275174185
  },
  "v2 vs v3|tasksource": {
   "n_common": 2206,
   "only_b_right": 619,
   "only_a_right": 376,
   "p_mcnemar_exact": 1.2590765149138642e-14
  },
  "v2 vs v3pre|tasksource": {
   "n_common": 2206,
   "only_b_right": 651,
   "only_a_right": 383,
   "p_mcnemar_exact": 6.8319004631405e-17
  },
  "v3 vs v3pre|tasksource": {
   "n_common": 2206,
   "only_b_right": 238,
   "only_a_right": 213,
   "p_mcnemar_exact": 0.25840324588421176
  },
  "v2 vs v2-engram_off|tasksource": {
   "n_common": 2206,
   "only_b_right": 117,
   "only_a_right": 151,
   "p_mcnemar_exact": 0.04361997977786522
  },
  "v3 vs v3-engram_off|tasksource": {
   "n_common": 2206,
   "only_b_right": 178,
   "only_a_right": 234,
   "p_mcnemar_exact": 0.00666462862779683
  },
  "v3pre vs v3pre-engram_off|tasksource": {
   "n_common": 2206,
   "only_b_right": 187,
   "only_a_right": 222,
   "p_mcnemar_exact": 0.09260412374435281
  }
 },
 "typed_en_by_teacher_agreement": {
  "v2": {
   "split": {
    "n": 812,
    "accuracy": 0.6059113300492611
   },
   "agreed": {
    "n": 1188,
    "accuracy": 0.8569023569023569
   }
  },
  "v2-engram_off": {
   "split": {
    "n": 812,
    "accuracy": 0.6009852216748769
   },
   "agreed": {
    "n": 1188,
    "accuracy": 0.8341750841750841
   }
  },
  "v3": {
   "split": {
    "n": 812,
    "accuracy": 0.6169950738916257
   },
   "agreed": {
    "n": 1188,
    "accuracy": 0.8644781144781145
   }
  },
  "v3-engram_off": {
   "split": {
    "n": 812,
    "accuracy": 0.6009852216748769
   },
   "agreed": {
    "n": 1188,
    "accuracy": 0.8468013468013468
   }
  },
  "v3pre": {
   "split": {
    "n": 812,
    "accuracy": 0.5874384236453202
   },
   "agreed": {
    "n": 1188,
    "accuracy": 0.8198653198653199
   }
  },
  "v3pre-engram_off": {
   "split": {
    "n": 812,
    "accuracy": 0.5455665024630542
   },
   "agreed": {
    "n": 1188,
    "accuracy": 0.7769360269360269
   }
  }
 },
 "tasksource": {
  "v2": {
   "by_family": {
    "CONDAQA": [
     3,
     10
    ],
    "CREAK": [
     5,
     10
    ],
    "FLUTE": [
     6,
     10
    ],
    "FOL-nli": [
     3,
     10
    ],
    "HatemojiBuild": [
     3,
     10
    ],
    "HelpSteer": [
     4,
     14
    ],
    "HelpSteer2": [
     8,
     26
    ],
    "HelpSteer3": [
     15,
     48
    ],
    "I2D2": [
     4,
     10
    ],
    "MSciNLI": [
     4,
     10
    ],
    "MedQA-USMLE-4-options-hf": [
     1,
     10
    ],
    "PARADISE": [
     5,
     10
    ],
    "PARARULE-Plus": [
     6,
     10
    ],
    "Prompt-injection-dataset": [
     5,
     10
    ],
    "PromptShield": [
     4,
     10
    ],
    "SDOH-NLI": [
     8,
     10
    ],
    "SpaceNLI": [
     3,
     10
    ],
    "Touche23-ValueEval": [
     7,
     10
    ],
    "TroFi": [
     4,
     10
    ],
    "TuringBench": [
     2,
     10
    ],
    "UNLI": [
     3,
     10
    ],
    "UltraFeedback-paired": [
     7,
     10
    ],
    "WANLI": [
     5,
     10
    ],
    "acceptability-prediction": [
     7,
     20
    ],
    "agent_action_safety": [
     6,
     10
    ],
    "ai2_arc": [
     6,
     20
    ],
    "amazon_counterfactual": [
     4,
     10
    ],
    "amazon_polarity": [
     5,
     10
    ],
    "arct": [
     5,
     10
    ],
    "args_me": [
     6,
     10
    ],
    "autotnli": [
     5,
     10
    ],
    "babi_nli": [
     17,
     35
    ],
    "balanced-copa": [
     5,
     10
    ],
    "banking77": [
     1,
     2
    ],
    "blog_authorship_corpus": [
     7,
     18
    ],
    "breaking_nli": [
     2,
     10
    ],
    "cicero": [
     2,
     10
    ],
    "circa": [
     2,
     10
    ],
    "civil_comments": [
     60,
     64
    ],
    "cladder": [
     5,
     10
    ],
    "clcd-english": [
     5,
     10
    ],
    "cloth": [
     1,
     10
    ],
    "codah": [
     2,
     10
    ],
    "commonsense_qa": [
     3,
     10
    ],
    "commonsense_qa_2.0": [
     4,
     10
    ],
    "conceptrules_v2": [
     6,
     10
    ],
    "corr2cause": [
     3,
     10
    ],
    "cosmos_qa": [
     2,
     10
    ],
    "cycic_classification": [
     2,
     10
    ],
    "cycic_multiplechoice": [
     3,
     10
    ],
    "dbpedia_14": [
     0,
     10
    ],
    "defeasible-nli": [
     13,
     22
    ],
    "dgen": [
     2,
     10
    ],
    "discovery": [
     0,
     2
    ],
    "disrpt": [
     2,
     10
    ],
    "dnd_style_intents": [
     2,
     10
    ],
    "doc-nli": [
     5,
     10
    ],
    "dynahate": [
     5,
     10
    ],
    "dynasent": [
     6,
     20
    ],
    "e-CARE": [
     6,
     10
    ],
    "emo": [
     1,
     10
    ],
    "equate": [
     5,
     10
    ],
    "esci": [
     4,
     10
    ],
    "ethics": [
     12,
     22
    ],
    "ethos": [
     2,
     10
    ],
    "feasibilityQA": [
     2,
     10
    ],
    "fever-evidence-related": [
     7,
     10
    ],
    "fig-qa": [
     4,
     10
    ],
    "folio": [
     3,
     10
    ],
    "fool-me-twice": [
     7,
     10
    ],
    "gen_debiased_nli": [
     0,
     2
    ],
    "github-issue-similarity": [
     7,
     10
    ],
    "go_emotions": [
     0,
     1
    ],
    "goal-step-wikihow": [
     15,
     30
    ],
    "google_wellformed_query": [
     7,
     10
    ],
    "hate_speech18": [
     4,
     10
    ],
    "head_qa": [
     2,
     10
    ],
    "headline_cause": [
     2,
     9
    ],
    "health_fact": [
     3,
     10
    ],
    "help-nli": [
     6,
     10
    ],
    "hh-rlhf": [
     16,
     40
    ],
    "hlgd": [
     3,
     10
    ],
    "hope_edi": [
     9,
     10
    ],
    "hover": [
     5,
     10
    ],
    "hover-3way": [
     5,
     10
    ],
    "hyperpartisan_news": [
     1,
     9
    ],
    "implicatures": [
     7,
     10
    ],
    "it-support-tickets": [
     1,
     10
    ],
    "jigsaw_toxicity": [
     2,
     10
    ],
    "lex_glue": [
     8,
     30
    ],
    "logical-entailment": [
     6,
     10
    ],
    "lonli": [
     6,
     10
    ],
    "math_qa": [
     2,
     9
    ],
    "medmcqa": [
     3,
     10
    ],
    "mindgames": [
     7,
     10
    ],
    "monotonicity-entailment": [
     7,
     10
    ],
    "moral_stories": [
     4,
     10
    ],
    "multilingual": [
     111,
     390
    ],
    "nan-nli": [
     5,
     10
    ],
    "naturallogic": [
     1,
     10
    ],
    "numer_sense": [
     2,
     10
    ],
    "onestop_qa": [
     4,
     10
    ],
    "open_question_type": [
     1,
     10
    ],
    "patent-phrase-similarity": [
     2,
     10
    ],
    "poem_sentiment": [
     1,
     10
    ],
    "prm800k_dpo": [
     10,
     20
    ],
    "probability_words_nli": [
     11,
     23
    ],
    "prompt-injection-dataset": [
     7,
     10
    ],
    "prompt-injections": [
     3,
     10
    ],
    "propsegment": [
     3,
     10
    ],
    "prost": [
     3,
     10
    ],
    "proto_qa": [
     3,
     10
    ],
    "puzzte": [
     6,
     10
    ],
    "qasc": [
     1,
     10
    ],
    "quartz": [
     5,
     10
    ],
    "recast": [
     10,
     22
    ],
    "resnli": [
     6,
     10
    ],
    "robust_nli": [
     1,
     4
    ],
    "robust_nli_is_sd": [
     1,
     3
    ],
    "robust_nli_li_ts": [
     1,
     2
    ],
    "ruletaker": [
     4,
     10
    ],
    "rumoureval_2019": [
     8,
     10
    ],
    "scientific-exaggeration-detection": [
     4,
     10
    ],
    "scinli": [
     2,
     10
    ],
    "scone": [
     3,
     10
    ],
    "scruples": [
     3,
     10
    ],
    "sem_eval_2010_task_8": [
     4,
     10
    ],
    "shell-safety-v2": [
     3,
     10
    ],
    "sherliic": [
     5,
     10
    ],
    "silicone": [
     0,
     8
    ],
    "sms_spam": [
     2,
     10
    ],
    "snips_built_in_intents": [
     3,
     10
    ],
    "snli": [
     4,
     10
    ],
    "social_i_qa": [
     4,
     10
    ],
    "spartqa-mchoice": [
     3,
     10
    ],
    "spartqa-yn": [
     4,
     10
    ],
    "stackoverflow-questions": [
     2,
     10
    ],
    "starcon": [
     8,
     10
    ],
    "stepgame": [
     2,
     10
    ],
    "sts-companion": [
     2,
     10
    ],
    "subjectivity": [
     6,
     10
    ],
    "syntactic-augmentation-nli": [
     2,
     10
    ],
    "temporal-nli": [
     3,
     10
    ],
    "tomi-nli": [
     7,
     10
    ],
    "tracie": [
     6,
     10
    ],
    "tweets_hate_speech_detection": [
     1,
     10
    ],
    "twitter-financial-news-sentiment": [
     1,
     10
    ],
    "utilitarianism": [
     4,
     10
    ],
    "vitaminc": [
     5,
     10
    ],
    "wice": [
     1,
     10
    ],
    "wiki_hop": [
     1,
     10
    ],
    "wikimedqa": [
     2,
     10
    ],
    "wildguardmix-cleaned": [
     10,
     21
    ],
    "winodict": [
     4,
     10
    ],
    "winowhy": [
     6,
     10
    ],
    "wnut_17": [
     2,
     10
    ],
    "wouldyourather": [
     4,
     10
    ]
   },
   "by_state_overlap": {
    "state_text_not_in_train": [
     845,
     2155
    ],
    "state_text_in_train": [
     5,
     51
    ]
   },
   "by_variant": {
    "direct": [
     692,
     1814
    ],
    "packed_derived": [
     66,
     164
    ],
    "label_verification": [
     42,
     87
    ],
    "instruction_paraphrase": [
     8,
     40
    ],
    "criteria_permutation": [
     35,
     80
    ],
    "paired_text_format": [
     7,
     21
    ]
   }
  },
  "v3": {
   "by_family": {
    "CONDAQA": [
     4,
     10
    ],
    "CREAK": [
     7,
     10
    ],
    "FLUTE": [
     6,
     10
    ],
    "FOL-nli": [
     7,
     10
    ],
    "HatemojiBuild": [
     5,
     10
    ],
    "HelpSteer": [
     6,
     14
    ],
    "HelpSteer2": [
     15,
     26
    ],
    "HelpSteer3": [
     16,
     48
    ],
    "I2D2": [
     6,
     10
    ],
    "MSciNLI": [
     1,
     10
    ],
    "MedQA-USMLE-4-options-hf": [
     5,
     10
    ],
    "PARADISE": [
     8,
     10
    ],
    "PARARULE-Plus": [
     7,
     10
    ],
    "Prompt-injection-dataset": [
     10,
     10
    ],
    "PromptShield": [
     7,
     10
    ],
    "SDOH-NLI": [
     9,
     10
    ],
    "SpaceNLI": [
     5,
     10
    ],
    "Touche23-ValueEval": [
     6,
     10
    ],
    "TroFi": [
     4,
     10
    ],
    "TuringBench": [
     1,
     10
    ],
    "UNLI": [
     6,
     10
    ],
    "UltraFeedback-paired": [
     4,
     10
    ],
    "WANLI": [
     2,
     10
    ],
    "acceptability-prediction": [
     8,
     20
    ],
    "agent_action_safety": [
     4,
     10
    ],
    "ai2_arc": [
     12,
     20
    ],
    "amazon_counterfactual": [
     7,
     10
    ],
    "amazon_polarity": [
     7,
     10
    ],
    "arct": [
     4,
     10
    ],
    "args_me": [
     4,
     10
    ],
    "autotnli": [
     5,
     10
    ],
    "babi_nli": [
     18,
     35
    ],
    "balanced-copa": [
     7,
     10
    ],
    "banking77": [
     2,
     2
    ],
    "blog_authorship_corpus": [
     6,
     18
    ],
    "breaking_nli": [
     7,
     10
    ],
    "cicero": [
     7,
     10
    ],
    "circa": [
     5,
     10
    ],
    "civil_comments": [
     61,
     64
    ],
    "cladder": [
     3,
     10
    ],
    "clcd-english": [
     4,
     10
    ],
    "cloth": [
     4,
     10
    ],
    "codah": [
     1,
     10
    ],
    "commonsense_qa": [
     1,
     10
    ],
    "commonsense_qa_2.0": [
     5,
     10
    ],
    "conceptrules_v2": [
     3,
     10
    ],
    "corr2cause": [
     7,
     10
    ],
    "cosmos_qa": [
     3,
     10
    ],
    "cycic_classification": [
     5,
     10
    ],
    "cycic_multiplechoice": [
     2,
     10
    ],
    "dbpedia_14": [
     9,
     10
    ],
    "defeasible-nli": [
     13,
     22
    ],
    "dgen": [
     6,
     10
    ],
    "discovery": [
     1,
     2
    ],
    "disrpt": [
     2,
     10
    ],
    "dnd_style_intents": [
     3,
     10
    ],
    "doc-nli": [
     5,
     10
    ],
    "dynahate": [
     1,
     10
    ],
    "dynasent": [
     11,
     20
    ],
    "e-CARE": [
     7,
     10
    ],
    "emo": [
     7,
     10
    ],
    "equate": [
     3,
     10
    ],
    "esci": [
     9,
     10
    ],
    "ethics": [
     11,
     22
    ],
    "ethos": [
     4,
     10
    ],
    "feasibilityQA": [
     4,
     10
    ],
    "fever-evidence-related": [
     9,
     10
    ],
    "fig-qa": [
     5,
     10
    ],
    "folio": [
     4,
     10
    ],
    "fool-me-twice": [
     7,
     10
    ],
    "gen_debiased_nli": [
     2,
     2
    ],
    "github-issue-similarity": [
     6,
     10
    ],
    "go_emotions": [
     1,
     1
    ],
    "goal-step-wikihow": [
     16,
     30
    ],
    "google_wellformed_query": [
     7,
     10
    ],
    "hate_speech18": [
     10,
     10
    ],
    "head_qa": [
     4,
     10
    ],
    "headline_cause": [
     7,
     9
    ],
    "health_fact": [
     4,
     10
    ],
    "help-nli": [
     6,
     10
    ],
    "hh-rlhf": [
     29,
     40
    ],
    "hlgd": [
     8,
     10
    ],
    "hope_edi": [
     8,
     10
    ],
    "hover": [
     6,
     10
    ],
    "hover-3way": [
     3,
     10
    ],
    "hyperpartisan_news": [
     6,
     9
    ],
    "implicatures": [
     4,
     10
    ],
    "it-support-tickets": [
     5,
     10
    ],
    "jigsaw_toxicity": [
     8,
     10
    ],
    "lex_glue": [
     17,
     30
    ],
    "logical-entailment": [
     6,
     10
    ],
    "lonli": [
     1,
     10
    ],
    "math_qa": [
     3,
     9
    ],
    "medmcqa": [
     1,
     10
    ],
    "mindgames": [
     8,
     10
    ],
    "monotonicity-entailment": [
     6,
     10
    ],
    "moral_stories": [
     6,
     10
    ],
    "multilingual": [
     162,
     390
    ],
    "nan-nli": [
     6,
     10
    ],
    "naturallogic": [
     3,
     10
    ],
    "numer_sense": [
     0,
     10
    ],
    "onestop_qa": [
     3,
     10
    ],
    "open_question_type": [
     7,
     10
    ],
    "patent-phrase-similarity": [
     4,
     10
    ],
    "poem_sentiment": [
     8,
     10
    ],
    "prm800k_dpo": [
     11,
     20
    ],
    "probability_words_nli": [
     13,
     23
    ],
    "prompt-injection-dataset": [
     5,
     10
    ],
    "prompt-injections": [
     4,
     10
    ],
    "propsegment": [
     6,
     10
    ],
    "prost": [
     4,
     10
    ],
    "proto_qa": [
     3,
     10
    ],
    "puzzte": [
     4,
     10
    ],
    "qasc": [
     6,
     10
    ],
    "quartz": [
     6,
     10
    ],
    "recast": [
     12,
     22
    ],
    "resnli": [
     5,
     10
    ],
    "robust_nli": [
     2,
     4
    ],
    "robust_nli_is_sd": [
     2,
     3
    ],
    "robust_nli_li_ts": [
     1,
     2
    ],
    "ruletaker": [
     4,
     10
    ],
    "rumoureval_2019": [
     5,
     10
    ],
    "scientific-exaggeration-detection": [
     4,
     10
    ],
    "scinli": [
     2,
     10
    ],
    "scone": [
     4,
     10
    ],
    "scruples": [
     4,
     10
    ],
    "sem_eval_2010_task_8": [
     1,
     10
    ],
    "shell-safety-v2": [
     4,
     10
    ],
    "sherliic": [
     6,
     10
    ],
    "silicone": [
     1,
     8
    ],
    "sms_spam": [
     10,
     10
    ],
    "snips_built_in_intents": [
     6,
     10
    ],
    "snli": [
     4,
     10
    ],
    "social_i_qa": [
     4,
     10
    ],
    "spartqa-mchoice": [
     1,
     10
    ],
    "spartqa-yn": [
     5,
     10
    ],
    "stackoverflow-questions": [
     3,
     10
    ],
    "starcon": [
     5,
     10
    ],
    "stepgame": [
     1,
     10
    ],
    "sts-companion": [
     4,
     10
    ],
    "subjectivity": [
     5,
     10
    ],
    "syntactic-augmentation-nli": [
     2,
     10
    ],
    "temporal-nli": [
     5,
     10
    ],
    "tomi-nli": [
     6,
     10
    ],
    "tracie": [
     5,
     10
    ],
    "tweets_hate_speech_detection": [
     9,
     10
    ],
    "twitter-financial-news-sentiment": [
     5,
     10
    ],
    "utilitarianism": [
     5,
     10
    ],
    "vitaminc": [
     2,
     10
    ],
    "wice": [
     7,
     10
    ],
    "wiki_hop": [
     1,
     10
    ],
    "wikimedqa": [
     2,
     10
    ],
    "wildguardmix-cleaned": [
     9,
     21
    ],
    "winodict": [
     4,
     10
    ],
    "winowhy": [
     4,
     10
    ],
    "wnut_17": [
     3,
     10
    ],
    "wouldyourather": [
     6,
     10
    ]
   },
   "by_state_overlap": {
    "state_text_not_in_train": [
     1080,
     2155
    ],
    "state_text_in_train": [
     13,
     51
    ]
   },
   "by_variant": {
    "direct": [
     896,
     1814
    ],
    "packed_derived": [
     76,
     164
    ],
    "label_verification": [
     49,
     87
    ],
    "instruction_paraphrase": [
     19,
     40
    ],
    "criteria_permutation": [
     42,
     80
    ],
    "paired_text_format": [
     11,
     21
    ]
   }
  },
  "v3pre": {
   "by_family": {
    "CONDAQA": [
     6,
     10
    ],
    "CREAK": [
     6,
     10
    ],
    "FLUTE": [
     4,
     10
    ],
    "FOL-nli": [
     4,
     10
    ],
    "HatemojiBuild": [
     6,
     10
    ],
    "HelpSteer": [
     6,
     14
    ],
    "HelpSteer2": [
     16,
     26
    ],
    "HelpSteer3": [
     16,
     48
    ],
    "I2D2": [
     4,
     10
    ],
    "MSciNLI": [
     3,
     10
    ],
    "MedQA-USMLE-4-options-hf": [
     3,
     10
    ],
    "PARADISE": [
     7,
     10
    ],
    "PARARULE-Plus": [
     5,
     10
    ],
    "Prompt-injection-dataset": [
     5,
     10
    ],
    "PromptShield": [
     7,
     10
    ],
    "SDOH-NLI": [
     9,
     10
    ],
    "SpaceNLI": [
     5,
     10
    ],
    "Touche23-ValueEval": [
     5,
     10
    ],
    "TroFi": [
     4,
     10
    ],
    "TuringBench": [
     1,
     10
    ],
    "UNLI": [
     5,
     10
    ],
    "UltraFeedback-paired": [
     6,
     10
    ],
    "WANLI": [
     3,
     10
    ],
    "acceptability-prediction": [
     9,
     20
    ],
    "agent_action_safety": [
     4,
     10
    ],
    "ai2_arc": [
     14,
     20
    ],
    "amazon_counterfactual": [
     8,
     10
    ],
    "amazon_polarity": [
     6,
     10
    ],
    "arct": [
     5,
     10
    ],
    "args_me": [
     6,
     10
    ],
    "autotnli": [
     6,
     10
    ],
    "babi_nli": [
     15,
     35
    ],
    "balanced-copa": [
     10,
     10
    ],
    "banking77": [
     2,
     2
    ],
    "blog_authorship_corpus": [
     9,
     18
    ],
    "breaking_nli": [
     9,
     10
    ],
    "cicero": [
     7,
     10
    ],
    "circa": [
     6,
     10
    ],
    "civil_comments": [
     61,
     64
    ],
    "cladder": [
     6,
     10
    ],
    "clcd-english": [
     6,
     10
    ],
    "cloth": [
     2,
     10
    ],
    "codah": [
     1,
     10
    ],
    "commonsense_qa": [
     1,
     10
    ],
    "commonsense_qa_2.0": [
     5,
     10
    ],
    "conceptrules_v2": [
     6,
     10
    ],
    "corr2cause": [
     7,
     10
    ],
    "cosmos_qa": [
     2,
     10
    ],
    "cycic_classification": [
     5,
     10
    ],
    "cycic_multiplechoice": [
     2,
     10
    ],
    "dbpedia_14": [
     8,
     10
    ],
    "defeasible-nli": [
     13,
     22
    ],
    "dgen": [
     8,
     10
    ],
    "discovery": [
     1,
     2
    ],
    "disrpt": [
     2,
     10
    ],
    "dnd_style_intents": [
     3,
     10
    ],
    "doc-nli": [
     6,
     10
    ],
    "dynahate": [
     3,
     10
    ],
    "dynasent": [
     12,
     20
    ],
    "e-CARE": [
     7,
     10
    ],
    "emo": [
     6,
     10
    ],
    "equate": [
     3,
     10
    ],
    "esci": [
     9,
     10
    ],
    "ethics": [
     11,
     22
    ],
    "ethos": [
     8,
     10
    ],
    "feasibilityQA": [
     9,
     10
    ],
    "fever-evidence-related": [
     7,
     10
    ],
    "fig-qa": [
     6,
     10
    ],
    "folio": [
     2,
     10
    ],
    "fool-me-twice": [
     7,
     10
    ],
    "gen_debiased_nli": [
     1,
     2
    ],
    "github-issue-similarity": [
     7,
     10
    ],
    "go_emotions": [
     1,
     1
    ],
    "goal-step-wikihow": [
     15,
     30
    ],
    "google_wellformed_query": [
     4,
     10
    ],
    "hate_speech18": [
     10,
     10
    ],
    "head_qa": [
     3,
     10
    ],
    "headline_cause": [
     7,
     9
    ],
    "health_fact": [
     4,
     10
    ],
    "help-nli": [
     5,
     10
    ],
    "hh-rlhf": [
     28,
     40
    ],
    "hlgd": [
     8,
     10
    ],
    "hope_edi": [
     9,
     10
    ],
    "hover": [
     6,
     10
    ],
    "hover-3way": [
     2,
     10
    ],
    "hyperpartisan_news": [
     4,
     9
    ],
    "implicatures": [
     4,
     10
    ],
    "it-support-tickets": [
     5,
     10
    ],
    "jigsaw_toxicity": [
     8,
     10
    ],
    "lex_glue": [
     20,
     30
    ],
    "logical-entailment": [
     7,
     10
    ],
    "lonli": [
     2,
     10
    ],
    "math_qa": [
     3,
     9
    ],
    "medmcqa": [
     2,
     10
    ],
    "mindgames": [
     9,
     10
    ],
    "monotonicity-entailment": [
     5,
     10
    ],
    "moral_stories": [
     6,
     10
    ],
    "multilingual": [
     169,
     390
    ],
    "nan-nli": [
     3,
     10
    ],
    "naturallogic": [
     2,
     10
    ],
    "numer_sense": [
     1,
     10
    ],
    "onestop_qa": [
     3,
     10
    ],
    "open_question_type": [
     4,
     10
    ],
    "patent-phrase-similarity": [
     3,
     10
    ],
    "poem_sentiment": [
     10,
     10
    ],
    "prm800k_dpo": [
     11,
     20
    ],
    "probability_words_nli": [
     13,
     23
    ],
    "prompt-injection-dataset": [
     5,
     10
    ],
    "prompt-injections": [
     5,
     10
    ],
    "propsegment": [
     4,
     10
    ],
    "prost": [
     4,
     10
    ],
    "proto_qa": [
     4,
     10
    ],
    "puzzte": [
     5,
     10
    ],
    "qasc": [
     7,
     10
    ],
    "quartz": [
     7,
     10
    ],
    "recast": [
     8,
     22
    ],
    "resnli": [
     7,
     10
    ],
    "robust_nli": [
     2,
     4
    ],
    "robust_nli_is_sd": [
     1,
     3
    ],
    "robust_nli_li_ts": [
     1,
     2
    ],
    "ruletaker": [
     3,
     10
    ],
    "rumoureval_2019": [
     6,
     10
    ],
    "scientific-exaggeration-detection": [
     5,
     10
    ],
    "scinli": [
     4,
     10
    ],
    "scone": [
     4,
     10
    ],
    "scruples": [
     4,
     10
    ],
    "sem_eval_2010_task_8": [
     2,
     10
    ],
    "shell-safety-v2": [
     5,
     10
    ],
    "sherliic": [
     6,
     10
    ],
    "silicone": [
     0,
     8
    ],
    "sms_spam": [
     9,
     10
    ],
    "snips_built_in_intents": [
     8,
     10
    ],
    "snli": [
     3,
     10
    ],
    "social_i_qa": [
     2,
     10
    ],
    "spartqa-mchoice": [
     3,
     10
    ],
    "spartqa-yn": [
     5,
     10
    ],
    "stackoverflow-questions": [
     3,
     10
    ],
    "starcon": [
     6,
     10
    ],
    "stepgame": [
     2,
     10
    ],
    "sts-companion": [
     4,
     10
    ],
    "subjectivity": [
     5,
     10
    ],
    "syntactic-augmentation-nli": [
     2,
     10
    ],
    "temporal-nli": [
     2,
     10
    ],
    "tomi-nli": [
     6,
     10
    ],
    "tracie": [
     2,
     10
    ],
    "tweets_hate_speech_detection": [
     9,
     10
    ],
    "twitter-financial-news-sentiment": [
     5,
     10
    ],
    "utilitarianism": [
     8,
     10
    ],
    "vitaminc": [
     2,
     10
    ],
    "wice": [
     7,
     10
    ],
    "wiki_hop": [
     1,
     10
    ],
    "wikimedqa": [
     3,
     10
    ],
    "wildguardmix-cleaned": [
     14,
     21
    ],
    "winodict": [
     4,
     10
    ],
    "winowhy": [
     4,
     10
    ],
    "wnut_17": [
     5,
     10
    ],
    "wouldyourather": [
     4,
     10
    ]
   },
   "by_state_overlap": {
    "state_text_not_in_train": [
     1102,
     2155
    ],
    "state_text_in_train": [
     16,
     51
    ]
   },
   "by_variant": {
    "direct": [
     924,
     1814
    ],
    "packed_derived": [
     82,
     164
    ],
    "label_verification": [
     48,
     87
    ],
    "instruction_paraphrase": [
     18,
     40
    ],
    "criteria_permutation": [
     35,
     80
    ],
    "paired_text_format": [
     11,
     21
    ]
   }
  }
 }
}