| { |
| "checkpoint": "cooldown from step 19000, step 8000", |
| "result": { |
| "ckpt": "step_008000.pt", |
| "step": 8000, |
| "val": { |
| "fineweb_edu": 2.74267241358757, |
| "wikipedia": 2.5070884823799133, |
| "cosmopedia": 1.930996522307396, |
| "dclm": 3.2088113725185394, |
| "stackexchange": 2.455552712082863, |
| "finemath": 3.174513578414917, |
| "code_python": 1.6154377311468124, |
| "overall": 2.572257514446974 |
| }, |
| "bench": { |
| "arc_challenge": 29.4, |
| "arc_easy": 56.4, |
| "hellaswag": 31.8, |
| "openbookqa": 30.2, |
| "piqa": 64.4, |
| "sciq": 73.0, |
| "winogrande": 53.8 |
| }, |
| "bench_avg": 48.42857142857143, |
| "blimp": { |
| "per_paradigm": { |
| "adjunct_island": 84.7, |
| "anaphor_gender_agreement": 94.8, |
| "anaphor_number_agreement": 98.6, |
| "animate_subject_passive": 77.2, |
| "animate_subject_trans": 87.2, |
| "causative": 73.9, |
| "complex_NP_island": 58.3, |
| "coordinate_structure_constraint_complex_left_branch": 58.1, |
| "coordinate_structure_constraint_object_extraction": 78.8, |
| "determiner_noun_agreement_1": 97.9, |
| "determiner_noun_agreement_2": 97.8, |
| "determiner_noun_agreement_irregular_1": 93.1, |
| "determiner_noun_agreement_irregular_2": 97.0, |
| "determiner_noun_agreement_with_adj_2": 94.8, |
| "determiner_noun_agreement_with_adj_irregular_1": 84.4, |
| "determiner_noun_agreement_with_adj_irregular_2": 92.4, |
| "determiner_noun_agreement_with_adjective_1": 94.9, |
| "distractor_agreement_relational_noun": 87.0, |
| "distractor_agreement_relative_clause": 72.6, |
| "drop_argument": 75.7, |
| "ellipsis_n_bar_1": 77.1, |
| "ellipsis_n_bar_2": 89.7, |
| "existential_there_object_raising": 82.7, |
| "existential_there_quantifiers_1": 98.9, |
| "existential_there_quantifiers_2": 52.7, |
| "existential_there_subject_raising": 83.8, |
| "expletive_it_object_raising": 75.8, |
| "inchoative": 67.6, |
| "intransitive": 78.1, |
| "irregular_past_participle_adjectives": 93.9, |
| "irregular_past_participle_verbs": 86.7, |
| "irregular_plural_subject_verb_agreement_1": 88.7, |
| "irregular_plural_subject_verb_agreement_2": 90.9, |
| "left_branch_island_echo_question": 42.7, |
| "left_branch_island_simple_question": 63.2, |
| "matrix_question_npi_licensor_present": 54.8, |
| "npi_present_1": 62.0, |
| "npi_present_2": 60.0, |
| "only_npi_licensor_present": 87.4, |
| "only_npi_scope": 68.2, |
| "passive_1": 88.3, |
| "passive_2": 88.7, |
| "principle_A_c_command": 56.5, |
| "principle_A_case_1": 100.0, |
| "principle_A_case_2": 95.8, |
| "principle_A_domain_1": 96.3, |
| "principle_A_domain_2": 72.9, |
| "principle_A_domain_3": 59.0, |
| "principle_A_reconstruction": 35.3, |
| "regular_plural_subject_verb_agreement_1": 94.7, |
| "regular_plural_subject_verb_agreement_2": 90.4, |
| "sentential_negation_npi_licensor_present": 88.9, |
| "sentential_negation_npi_scope": 47.6, |
| "sentential_subject_island": 31.1, |
| "superlative_quantifiers_1": 97.2, |
| "superlative_quantifiers_2": 88.0, |
| "tough_vs_raising_1": 64.8, |
| "tough_vs_raising_2": 88.3, |
| "transitive": 87.6, |
| "wh_island": 73.9, |
| "wh_questions_object_gap": 76.3, |
| "wh_questions_subject_gap": 90.1, |
| "wh_questions_subject_gap_long_distance": 81.5, |
| "wh_vs_that_no_gap": 96.1, |
| "wh_vs_that_no_gap_long_distance": 94.5, |
| "wh_vs_that_with_gap": 52.6, |
| "wh_vs_that_with_gap_long_distance": 34.0 |
| }, |
| "paradigm_avg": 78.72388059701493, |
| "overall": 78.72388059701493, |
| "pairs": 67000 |
| }, |
| "seconds": 127 |
| }, |
| "comparison": { |
| "EleutherAI/gpt-neo-125m": { |
| "bench_avg": 43.94285714285714, |
| "blimp": 79.25820895522388, |
| "bench": { |
| "arc_challenge": 25.2, |
| "arc_easy": 41.8, |
| "hellaswag": 27.8, |
| "openbookqa": 25.6, |
| "piqa": 63.4, |
| "sciq": 73.4, |
| "winogrande": 50.4 |
| } |
| }, |
| "facebook/opt-125m": { |
| "bench_avg": 43.628571428571426, |
| "blimp": 81.40597014925373, |
| "bench": { |
| "arc_challenge": 25.4, |
| "arc_easy": 39.8, |
| "hellaswag": 30.6, |
| "openbookqa": 27.8, |
| "piqa": 61.0, |
| "sciq": 70.0, |
| "winogrande": 50.8 |
| } |
| }, |
| "openai-community/gpt2": { |
| "bench_avg": 43.114285714285714, |
| "blimp": 82.51194029850747, |
| "bench": { |
| "arc_challenge": 24.8, |
| "arc_easy": 39.6, |
| "hellaswag": 29.6, |
| "openbookqa": 25.8, |
| "piqa": 64.2, |
| "sciq": 67.6, |
| "winogrande": 50.2 |
| } |
| }, |
| "EleutherAI/pythia-160m": { |
| "bench_avg": 42.74285714285714, |
| "blimp": 78.52835820895523, |
| "bench": { |
| "arc_challenge": 25.4, |
| "arc_easy": 41.8, |
| "hellaswag": 26.0, |
| "openbookqa": 26.4, |
| "piqa": 62.8, |
| "sciq": 69.0, |
| "winogrande": 47.8 |
| } |
| }, |
| "distilbert/distilgpt2": { |
| "bench_avg": 40.94285714285714, |
| "blimp": 79.53283582089553, |
| "bench": { |
| "arc_challenge": 25.2, |
| "arc_easy": 36.0, |
| "hellaswag": 25.4, |
| "openbookqa": 26.0, |
| "piqa": 59.0, |
| "sciq": 64.8, |
| "winogrande": 50.2 |
| } |
| }, |
| "openai-community/openai-gpt": { |
| "bench_avg": 39.714285714285715, |
| "blimp": 77.5910447761194, |
| "bench": { |
| "arc_challenge": 25.8, |
| "arc_easy": 34.4, |
| "hellaswag": 22.8, |
| "openbookqa": 26.6, |
| "piqa": 55.8, |
| "sciq": 59.0, |
| "winogrande": 53.6 |
| } |
| } |
| } |
| } |
