Add m26 weights and results

m26/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m26",
+  "checkpoint": "results/m26/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 595.4815310921231,
+      "perplexity_stderr,none": 32.89384889135734,
+      "acc,none": 0.17950708325247428,
+      "acc_stderr,none": 0.0053467550223617434
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.27165903206532566,
+      "acc_stderr,none": 0.004439059440526545,
+      "acc_norm,none": 0.27823142800239,
+      "acc_norm_stderr,none": 0.004472121485161612
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.5979325353645266,
+      "acc_stderr,none": 0.011439867127267474,
+      "acc_norm,none": 0.5816104461371056,
+      "acc_norm_stderr,none": 0.011509380214588163
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.663,
+      "acc_stderr,none": 0.01495508791865346,
+      "acc_norm,none": 0.579,
+      "acc_norm_stderr,none": 0.015620595475301287
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.4511784511784512,
+      "acc_stderr,none": 0.010210757101073465,
+      "acc_norm,none": 0.4010942760942761,
+      "acc_norm_stderr,none": 0.010057051106534466
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5303867403314917,
+      "acc_stderr,none": 0.014026510839428848
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.773,
+      "acc_stderr,none": 0.013253174964763977
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.931,
+      "acc_stderr,none": 0.008018934050315124
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.985,
+      "acc_stderr,none": 0.003845749574502972
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.711,
+      "acc_stderr,none": 0.014341711358296287
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.903,
+      "acc_stderr,none": 0.009363689373248062
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.705,
+      "acc_stderr,none": 0.014428554438445425
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.437,
+      "acc_stderr,none": 0.015693223928730425
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.468,
+      "acc_stderr,none": 0.015786868759359002
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.738,
+      "acc_stderr,none": 0.013912208651021217
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.962,
+      "acc_stderr,none": 0.0060491811505849775
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.95,
+      "acc_stderr,none": 0.006895472974897918
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.898,
+      "acc_stderr,none": 0.009575368801653944
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.893,
+      "acc_stderr,none": 0.009779910359847203
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.91,
+      "acc_stderr,none": 0.009054390204866477
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.839,
+      "acc_stderr,none": 0.011628164696727263
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.886,
+      "acc_stderr,none": 0.010055103435823278
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.912,
+      "acc_stderr,none": 0.008963053962592012
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.847,
+      "acc_stderr,none": 0.011389500459665504
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.722,
+      "acc_stderr,none": 0.014174516461485272
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.766,
+      "acc_stderr,none": 0.01339490288966006
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.752,
+      "acc_stderr,none": 0.013663187134877524
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.879,
+      "acc_stderr,none": 0.010318210380946179
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.779,
+      "acc_stderr,none": 0.01312750285969618
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.97,
+      "acc_stderr,none": 0.0053971408290991564
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.35,
+      "acc_stderr,none": 0.015090650341444127
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.892,
+      "acc_stderr,none": 0.009820001651345708
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.766,
+      "acc_stderr,none": 0.01339490288966006
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.673,
+      "acc_stderr,none": 0.014842213153411162
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.742,
+      "acc_stderr,none": 0.013842963108656601
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.911,
+      "acc_stderr,none": 0.009008893392651516
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.794,
+      "acc_stderr,none": 0.012795613612786522
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.875,
+      "acc_stderr,none": 0.010463483381956722
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.883,
+      "acc_stderr,none": 0.010169287802713345
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.548,
+      "acc_stderr,none": 0.01574623586588058
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.536,
+      "acc_stderr,none": 0.015778243024904673
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.566,
+      "acc_stderr,none": 0.01568087656637504
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.54,
+      "acc_stderr,none": 0.0157685969143944
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.669,
+      "acc_stderr,none": 0.0148882725882038
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.894,
+      "acc_stderr,none": 0.00973955126578524
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.638,
+      "acc_stderr,none": 0.015204840912919383
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.844,
+      "acc_stderr,none": 0.011480235006122295
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.844,
+      "acc_stderr,none": 0.011480235006122295
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.58,
+      "acc_stderr,none": 0.015615500115072966
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.887,
+      "acc_stderr,none": 0.010016552866696773
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.95,
+      "acc_stderr,none": 0.006895472974897918
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.746,
+      "acc_stderr,none": 0.013772206565168622
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.644,
+      "acc_stderr,none": 0.015149042659306678
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.649,
+      "acc_stderr,none": 0.015100563798316508
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.866,
+      "acc_stderr,none": 0.010777762298369738
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.874,
+      "acc_stderr,none": 0.010499249222407964
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.998,
+      "acc_stderr,none": 0.0014135055705578234
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.697,
+      "acc_stderr,none": 0.014539683710535193
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.513,
+      "acc_stderr,none": 0.015813952101896636
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.921,
+      "acc_stderr,none": 0.00853415677333337
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.691,
+      "acc_stderr,none": 0.014619600977206347
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.634,
+      "acc_stderr,none": 0.0152406127264056
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.853,
+      "acc_stderr,none": 0.011203415395160376
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.825,
+      "acc_stderr,none": 0.012021627157731998
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.818,
+      "acc_stderr,none": 0.0122075806376622
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.775,
+      "acc_stderr,none": 0.013211720158614833
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.941,
+      "acc_stderr,none": 0.007454835650406693
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.894,
+      "acc_stderr,none": 0.00973955126578524
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.976,
+      "acc_stderr,none": 0.004842256441727084
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.982,
+      "acc_stderr,none": 0.00420638724961144
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.455,
+      "acc_stderr,none": 0.015755101498347232
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.202,
+      "acc_stderr,none": 0.01270265158765513
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7754029850746268,
+      "acc_stderr,none": 0.0014728963305261126,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m26/config.json

@@ -0,0 +1,50 @@
+{
+  "model_id": "m26",
+  "run_name": "m26-mla-rope-dense-adamw-seed1338",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mla",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 256,
+  "pos_type": "rope",
+  "rope_base": 10000,
+  "ffn_type": "dense",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "adamw",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 4,
+  "grad_accum_steps": 16,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1338,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts",
+  "seed_group": "m09"
+}

m26/eval_results.json

@@ -0,0 +1,212 @@
+{
+  "model_id": "m26",
+  "checkpoint": "/home/akshat/codemindv2/results/m26/final.pt",
+  "trained_step": 15258,
+  "total_params": 61920256,
+  "active_params": 61920256,
+  "flops_per_forward": 123840512.0,
+  "val_loss": 3.4715208411216736,
+  "val_perplexity": 32.185654508491076,
+  "bits_per_token": 5.008345901828922,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 149555.9788809332,
+  "inference_tok_per_sec": 209.79346279569137,
+  "inference_tok_per_sec_kvcache": 206.45125406031903,
+  "kvcache_speedup": 0.9840690520532226,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 0.385,
+  "peak_train_vram_gb": 4.062,
+  "closeness_answer_ppl": 21.671,
+  "closeness_hit_rate": 0.4,
+  "closeness_token_f1": 0.017,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " the capital of the French Republic",
+      "answer_nll": 1.999,
+      "answer_ppl": 7.382,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " the capital of the United States",
+      "answer_nll": 2.4891,
+      "answer_ppl": 12.05,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the Sun.\nThe Sun",
+      "answer_nll": 2.6785,
+      "answer_ppl": 14.564,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen.\nThe hydrogen is",
+      "answer_nll": 0.9639,
+      "answer_ppl": 2.622,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the heat of the sun.",
+      "answer_nll": 3.7372,
+      "answer_ppl": 41.982,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the same.\nThe difference",
+      "answer_nll": 5.0274,
+      "answer_ppl": 152.529,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": " the sum of the two numbers",
+      "answer_nll": 4.1376,
+      "answer_ppl": 62.655,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " sky, and the moon rises",
+      "answer_nll": 2.375,
+      "answer_ppl": 10.751,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington.\nThe first president",
+      "answer_nll": 0.8138,
+      "answer_ppl": 2.257,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe speed of",
+      "answer_nll": 0.278,
+      "answer_ppl": 1.32,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides, and the triangle has",
+      "answer_nll": 1.224,
+      "answer_ppl": 3.401,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " the gold.\nThe symbol",
+      "answer_nll": 8.9285,
+      "answer_ppl": 7543.906,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " about 1.5 degrees Celsius",
+      "answer_nll": 5.8914,
+      "answer_ppl": 361.902,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare, a playwright and",
+      "answer_nll": 1.0939,
+      "answer_ppl": 2.986,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " sun.\nThe Earth is",
+      "answer_nll": 0.7814,
+      "answer_ppl": 2.184,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the plural of the plural of",
+      "answer_nll": 8.0899,
+      "answer_ppl": 3261.209,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " photosynthesis of plants.\n",
+      "answer_nll": 3.6089,
+      "answer_ppl": 36.926,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood to the brain, and",
+      "answer_nll": 1.7317,
+      "answer_ppl": 5.65,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " United States dollar.\nThe",
+      "answer_nll": 2.5945,
+      "answer_ppl": 13.39,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " giant tortoise.\nThe",
+      "answer_nll": 3.0755,
+      "answer_ppl": 21.661,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is to do a regular sleep. You can do this without the help of a doctor, so you can rest and enjoy the rest you have, and you'll wake up, get up, and enjoy the rest you have.\nThe most important thing about sleep is to get more sleep. When you can",
+    "Scientists have discovered that": "Scientists have discovered that, in many ways, the human brain is capable of detecting an electrical \"brain wave.\"\n\"If we can understand the phenomenon of the brain wave, that is the human brain,\" said H. F. F. O'Brien, one of the authors of this study. \"However, there",
+    "In the history of mathematics,": "In the history of mathematics, the term arithmetic is used to describe the arithmetic of numbers and of the whole. The term is used to describe the study of mathematics. The earliest known use of the word arithmetic, however, dates from the time of the Roman Empire in the West by way of the Latin languages, and the study of",
+    "Python code for prime number": "Python code for prime number 32, and the number of lines on the top of a line is represented by a symbol. The function also is used to represent the first and last lines of a line with a symbol of the number of points in the line.\nThis is the function of\n- the number of lines of a",
+    "The best way to learn a new language is": "The best way to learn a new language is to learn about each language. As soon as you notice new words or new words, you can talk to a group of people in a new language by using the \u201cA\u201d and \u201cB\u201d examples.\n- Start talking with a group of people in a new language by using"
+  }
+}

m26/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eac5eaada727bb0c36e287f5141b052eca7269d3e1f5220fe25278ac2cd422b6
+size 247691704