{
  "checkpoint": "cooldown from step 19000, step 8000",
  "result": {
    "ckpt": "step_008000.pt",
    "step": 8000,
    "val": {
      "fineweb_edu": 2.74267241358757,
      "wikipedia": 2.5070884823799133,
      "cosmopedia": 1.930996522307396,
      "dclm": 3.2088113725185394,
      "stackexchange": 2.455552712082863,
      "finemath": 3.174513578414917,
      "code_python": 1.6154377311468124,
      "overall": 2.572257514446974
    },
    "bench": {
      "arc_challenge": 29.4,
      "arc_easy": 56.4,
      "hellaswag": 31.8,
      "openbookqa": 30.2,
      "piqa": 64.4,
      "sciq": 73.0,
      "winogrande": 53.8
    },
    "bench_avg": 48.42857142857143,
    "blimp": {
      "per_paradigm": {
        "adjunct_island": 84.7,
        "anaphor_gender_agreement": 94.8,
        "anaphor_number_agreement": 98.6,
        "animate_subject_passive": 77.2,
        "animate_subject_trans": 87.2,
        "causative": 73.9,
        "complex_NP_island": 58.3,
        "coordinate_structure_constraint_complex_left_branch": 58.1,
        "coordinate_structure_constraint_object_extraction": 78.8,
        "determiner_noun_agreement_1": 97.9,
        "determiner_noun_agreement_2": 97.8,
        "determiner_noun_agreement_irregular_1": 93.1,
        "determiner_noun_agreement_irregular_2": 97.0,
        "determiner_noun_agreement_with_adj_2": 94.8,
        "determiner_noun_agreement_with_adj_irregular_1": 84.4,
        "determiner_noun_agreement_with_adj_irregular_2": 92.4,
        "determiner_noun_agreement_with_adjective_1": 94.9,
        "distractor_agreement_relational_noun": 87.0,
        "distractor_agreement_relative_clause": 72.6,
        "drop_argument": 75.7,
        "ellipsis_n_bar_1": 77.1,
        "ellipsis_n_bar_2": 89.7,
        "existential_there_object_raising": 82.7,
        "existential_there_quantifiers_1": 98.9,
        "existential_there_quantifiers_2": 52.7,
        "existential_there_subject_raising": 83.8,
        "expletive_it_object_raising": 75.8,
        "inchoative": 67.6,
        "intransitive": 78.1,
        "irregular_past_participle_adjectives": 93.9,
        "irregular_past_participle_verbs": 86.7,
        "irregular_plural_subject_verb_agreement_1": 88.7,
        "irregular_plural_subject_verb_agreement_2": 90.9,
        "left_branch_island_echo_question": 42.7,
        "left_branch_island_simple_question": 63.2,
        "matrix_question_npi_licensor_present": 54.8,
        "npi_present_1": 62.0,
        "npi_present_2": 60.0,
        "only_npi_licensor_present": 87.4,
        "only_npi_scope": 68.2,
        "passive_1": 88.3,
        "passive_2": 88.7,
        "principle_A_c_command": 56.5,
        "principle_A_case_1": 100.0,
        "principle_A_case_2": 95.8,
        "principle_A_domain_1": 96.3,
        "principle_A_domain_2": 72.9,
        "principle_A_domain_3": 59.0,
        "principle_A_reconstruction": 35.3,
        "regular_plural_subject_verb_agreement_1": 94.7,
        "regular_plural_subject_verb_agreement_2": 90.4,
        "sentential_negation_npi_licensor_present": 88.9,
        "sentential_negation_npi_scope": 47.6,
        "sentential_subject_island": 31.1,
        "superlative_quantifiers_1": 97.2,
        "superlative_quantifiers_2": 88.0,
        "tough_vs_raising_1": 64.8,
        "tough_vs_raising_2": 88.3,
        "transitive": 87.6,
        "wh_island": 73.9,
        "wh_questions_object_gap": 76.3,
        "wh_questions_subject_gap": 90.1,
        "wh_questions_subject_gap_long_distance": 81.5,
        "wh_vs_that_no_gap": 96.1,
        "wh_vs_that_no_gap_long_distance": 94.5,
        "wh_vs_that_with_gap": 52.6,
        "wh_vs_that_with_gap_long_distance": 34.0
      },
      "paradigm_avg": 78.72388059701493,
      "overall": 78.72388059701493,
      "pairs": 67000
    },
    "seconds": 127
  },
  "comparison": {
    "EleutherAI/gpt-neo-125m": {
      "bench_avg": 43.94285714285714,
      "blimp": 79.25820895522388,
      "bench": {
        "arc_challenge": 25.2,
        "arc_easy": 41.8,
        "hellaswag": 27.8,
        "openbookqa": 25.6,
        "piqa": 63.4,
        "sciq": 73.4,
        "winogrande": 50.4
      }
    },
    "facebook/opt-125m": {
      "bench_avg": 43.628571428571426,
      "blimp": 81.40597014925373,
      "bench": {
        "arc_challenge": 25.4,
        "arc_easy": 39.8,
        "hellaswag": 30.6,
        "openbookqa": 27.8,
        "piqa": 61.0,
        "sciq": 70.0,
        "winogrande": 50.8
      }
    },
    "openai-community/gpt2": {
      "bench_avg": 43.114285714285714,
      "blimp": 82.51194029850747,
      "bench": {
        "arc_challenge": 24.8,
        "arc_easy": 39.6,
        "hellaswag": 29.6,
        "openbookqa": 25.8,
        "piqa": 64.2,
        "sciq": 67.6,
        "winogrande": 50.2
      }
    },
    "EleutherAI/pythia-160m": {
      "bench_avg": 42.74285714285714,
      "blimp": 78.52835820895523,
      "bench": {
        "arc_challenge": 25.4,
        "arc_easy": 41.8,
        "hellaswag": 26.0,
        "openbookqa": 26.4,
        "piqa": 62.8,
        "sciq": 69.0,
        "winogrande": 47.8
      }
    },
    "distilbert/distilgpt2": {
      "bench_avg": 40.94285714285714,
      "blimp": 79.53283582089553,
      "bench": {
        "arc_challenge": 25.2,
        "arc_easy": 36.0,
        "hellaswag": 25.4,
        "openbookqa": 26.0,
        "piqa": 59.0,
        "sciq": 64.8,
        "winogrande": 50.2
      }
    },
    "openai-community/openai-gpt": {
      "bench_avg": 39.714285714285715,
      "blimp": 77.5910447761194,
      "bench": {
        "arc_challenge": 25.8,
        "arc_easy": 34.4,
        "hellaswag": 22.8,
        "openbookqa": 26.6,
        "piqa": 55.8,
        "sciq": 59.0,
        "winogrande": 53.6
      }
    }
  }
}