diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af222877bc8b043b59d88431074038a8b136b0d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58d32fa4afda0cf544b6da454cfbf508dc4af253 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b7a24f390d50da9ca46270f5a156f6b7c6f18d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..727cf4b6e30a1c91484ff8afdeb6a30f04af79bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbc77935aa4d4692a638506b2cd7faae8441f391 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a923b6b2a3f3abf9ed04d6538f4aa62845dd0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72758a56c4df5f38c667ed0ccd2fe348a24a8ba7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4491c51bc8e3e781c2df0fe1902a13993d9afd3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f9b95ed6da3974a58fbd81cd0b664271ea1a022 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_other +task: + - global_mmlu_full_ja_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27d09b8887a4138f9385c89b8e4ba57cd24979c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15e26a51a3ba91a7397adf477b151d7043a8841a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52c92423e7159ddeed225b000a50a1cb28ec8236 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5413c86d5602d40aa982466a1dbbdda5a5589e10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..276f214e794fcd316ec34659bd0082d0b5d491e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbc6846a790eba71d2f5b6d763a4f0fd75c5bc49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..138071048c21e33f974483b47ddab8eefe7be743 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d88d5685abdca8a4bc9a36b4759820fd79b6a922 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64cb2b9ee4f6840e17722208a2cb07f65d6a7e75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b88adf90587b18715066659e4c692c4017188e10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eef67cc7773890154ae12dfe0d716c3883bab38b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c90e5fb76b61a6a581b6526bb448b3c8a678747a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8318099ac99a420effc370650e14441de386fb77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ed5a6209bb1330861a4da399a63a4c87696901c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ec0ab8422873fbdebd63e138ea80294c747dab3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bdd9555d65c540bd4a6277853b7b8b2c6c407ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3abfd81b1ca3bb9a7e7833a10d8d67535b24a150 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..483161a60914cab158c8b824567d25261701aaf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..702092af446264025bc997c8b8ceca537626df03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b7ce92eece5a40aa9ef4bd2e10a9f970cba3362 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c68acb8e827ea1dc1415d8ddaed18ccb5b34b197 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1b91833201575b2dd8f1a049a04a1bc5826c208 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dee1f89e90044d9cec96388b8477ec723b1622b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3612a7eec71d04f51167b53c0d230fda75bbae37 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b70204fbcd7d0e89a99a532ab220f19151be54bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77ed3c97d7e0d49191119fd8158005e84e5d29a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8fbb261f7e6ac1b4e3785e466c29bbe7b4d6446 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58d4afccbebf5c4150409d701159ab2d4a30f8b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e664390f0bf96f91706a04f6b89edf2dd761bef6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf495ae2272d991ad46dc0b15e4cc4956bad00ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1349771effd8889c35bf139f78d1a82913f149a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b513ac4f22e577808d87f45140a3876f3dbdf14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81659bf774474bbeab8a6ef4f30030da1155ed82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e77694bdac7e0ffef6457e1e0ad2cd0a3673cb8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f322376dae0acca5b0c32bd649545ba7d225f35a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d58fb0e60c1990541e5b2eea85d74aef9c23b66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..238653619ffdfc30b0b61435434faff5fda992b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a044bf99c52e926965669c9341991a0dcc9f09ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b828e0e28bd24315cb2b295cd96463bcf22ba930 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7aafb6c4cc3d41d8d9e005f67f749d0964a56530 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0cf9905e29139af85f77696a79e20f768a1d183 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5ef36c18369bffbd5d74a457398b28fc5142ae0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..565439e6d7f3ef9380452f7f946813a4ec417972 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7d21bd64a5770ad38531599a3b0aae3ca33b120 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cc44c1c12a4e077cef78f08fc7815067a0d3190 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ebdb14a8c2a008b610419ee51476d8cb0402b75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ja_social_sciences_tasks +task: global_mmlu_full_ja_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6f833672faa6d6cc43225f43cae9089c0c1cbd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23e66e063640e906f4c31f7fded8b0ba7fbfad08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ja_humanities_tasks +task: global_mmlu_full_ja_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b9b419701bee34d47215496fd20cd53a95f4038 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ko +task: + - global_mmlu_full_ko_stem + - global_mmlu_full_ko_other + - global_mmlu_full_ko_social_sciences + - global_mmlu_full_ko_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7690643456b274d6a2f07a4693ceb2371eae4e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_humanities +task: + - global_mmlu_full_ko_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8990ae957ca2bf03544f30495f92d95d641e0e70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_other +task: + - global_mmlu_full_ko_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bbfad7f24acb3042b85c0f8825f51f6e298a6eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_social_sciences +task: + - global_mmlu_full_ko_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18b7f17bae7d6a67d027342661184672a0b95a32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_global_mmlu_full_ko_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ko_stem +task: + - global_mmlu_full_ko_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_ko_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_ko_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..11700a2611eea6353ec3405e5dbaaf3269d53fed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/_ko_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ko +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5959d7881bc0366bcabb2c0144f761d8c4858996 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebb90860fe61eea9a47a5f6b0561b09a3d194338 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..670846b327253b15226b264abfc3ad5f76e46d00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a44e430211b6f47e042802ebd01d7b9ca5d7c04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9e29697c08bf5e7be4aa89032581826e24fdc10 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc3644680d3e97f82d4fbc801fa0feca11136f68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2eb0f416f3f3b0db9da85a702cd0df96bc30fbfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..044f1eff5a42751ada4ff2d05feba727ea68415e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9929097ce772a874064b4376f4556b446b048a86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b78c24e6fa2419591a3c072b19dd7133a168aaa2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20c3fb20a57b1cbdc87e15aa8016e01632135551 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f95457249b51a31d715ed22a9b39a37d39ee93f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f79989750da6c136e7971c6afe2f0c7c5febaea0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c35ed7f98c8cd53855ccbcab54b64c62e0ce54 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1444a2495a807e9fc826d2b2f6624fd90956bb35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bec91b9913b00714dc15978b6f4dd69c546099a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cf3109238c6074386ea0d9a30abdd3b489661bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a5f7bd5403deb9e6ea8515c9c4e7c671f120635 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdaed574d0b464fdc393f49be834a5fdb06375fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..193a064c984f847b6dc679cffa6cc0223fa30738 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d2ad648afc5763de6fb05e614043a61939a3f0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a48b602dd0b762a16bac7801a26c8cef0b819fad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc9c20eb6a430de9c445b55a7310870948c6737c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e86a27fa5c2f44f2cbcbafbf9680a0032bb91016 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b947f7f3f451ac902ca4e9d5ec036efccf7b81c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9184ad9ced39c8a52377da4eab6b856875d8fa6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50b6a150332ca02078913fe01226fc1341944ef5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..974e3b03da1722e347b4e90c18762232238a830f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a0105961c9365ad342dbc006c8425ac868a67f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a696675df6951a6d882b89e6c0da31fa7fe78c82 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eca86cbe496ee634f735dd2d6b2daf3557320f7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed3e99fc51a7d31798eca2b3b18487fd92d4866f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01eec477c5c460e6334255f3463124604fd0e174 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cce25c098de00ed09758aa874658822776f0b7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04b71e2a702f13002c84198f4d531ab07fe91d45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..168cae74453a423c475b4ed7f13f0fc2f7b9cda0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d1ad959349ad14484c27b28a419b3a1081c407a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36cd7e20620780639fe050aff5facfc45dbade27 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a7791512f829437488c5fd8117acc96f75c618 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..563c1397689df77649e0a29c291ec3009d9d2210 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..527c71074d96038c1414a9d24ac60c45b4dde78e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c69754b32b3d7affd9a2313a8b2be9f40dbbaf2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d0085ab968a802e03eb40c8526d8d249d382ce6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e9a01030ee326f7a950704b589c8169beea6a67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec9a665b52d2a0c8aa1af6787eff4cb839c998ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d81a9470486926fd53c9cc4f9d2d8d8a8fc19eb1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..139376cc770a8a0c321ff3241147afff31e5b164 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87112d8c77823c53196ae5f1ec70e5d43cd9cfee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f365fab1d50d5cc33935198ab2f5e3d0a80d500 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3a6f9214771c7281150b390193cad51fe844c98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..526b68ed9051c96d53fe8c54f95c386173ccaacb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e14b1dcea2da09877e98827770e1b14fe7b55cb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cdf5c90b9cde143640c51d349fb23fef3f0ada7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2f2a2103bf5df058fbbb5540b83ec9b5216a955 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd363709173087d90cfbb9e65cf1df80361f74af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad65a13643885afa6d00742737983ff2376b4c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dd6d699a1a2eee445ba12047e4687c86f7e8440 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fb0ee1e513d860418cb68acb630cfc7ef1aa10d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75f2769ada0253604eec937509d7927532b0c377 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb27e9bd2471b6de27150e60d89df8fd6103d27 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9d952c39b0a13ff0887fdb671535776babe9088 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f77adf9b399d0dfbf28debccb427d95ab2467245 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6be84fde4ca8de4bc8d1795ad8ba6a223bb3d50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad597b27aa2717ae8aeec74c69a1595bfafe129a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb06a871b072676ed1e4d4e5c09faa9657346a22 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1885ad3faa748e02b78cd1691e8cf4f752da223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dc83089094d69a5872cd596374c7ef9d3707ac2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b67d321e40986fd04d93a75073868e98c7b0998f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c744613c6e76867f77f901428b119071474c06d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09e6f044fcf2202fcc6d8278649996613662b261 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb8dd3307fa10a0c5628413331545e7773c4d245 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b1a85566793f56997423ae6063b75b44aa017d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aab1d556c86ddddcc47ec278c45c6dff556641ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac93dd6cc9c14abe297b765b04a71c8b51a44ab4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6be78ec390713a42935600ead6cdf9734c8f5cd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60b6cdcc0b9016e13ed117af0ff9f38728f536cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd899676e6dfedcda15b8d30b048906b81983604 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd796e8b55c413a9c1937b92ad797174e8a0186b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c6e5f397293a0cda432923d9c9bb8d71b305af3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eb9957d2df677f8d81aa8cd4123671cef29c53e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e17f95a66b8d28a0b68e6050241d6140988b7d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d39bb63caec03b27a46a16c3094aacd15eb4d567 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_lt_social_sciences_tasks +task: global_mmlu_full_lt_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8482a618bda42c8917f010a31bc259dbb4386e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a86af60dce82c39b95efc1da3e430334d30c1df6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_lt_humanities_tasks +task: global_mmlu_full_lt_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcc6cd81adfab398e1164a063d6303218bbea53f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_mg +task: + - global_mmlu_full_mg_stem + - global_mmlu_full_mg_other + - global_mmlu_full_mg_social_sciences + - global_mmlu_full_mg_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76b08f6c776b956f312b05f879ba0a278025607f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_mg_humanities +task: + - global_mmlu_full_mg_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0006af4c995b0abfd5d34b87e363218b2a445c20 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_mg_other +task: + - global_mmlu_full_mg_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cfe4f5b02b6fb983c3ccb65f9a4345fa3b39222 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_mg_social_sciences +task: + - global_mmlu_full_mg_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdc719d1076ec7081b6547d655887bfb6c81dbc0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_global_mmlu_full_mg_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_mg_stem +task: + - global_mmlu_full_mg_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_mg_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_mg_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4aa97b27f3b6926ba42070c81c64b004e8f331ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/_mg_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: mg +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bea850ed14165bc4e32806c9828f2074b86ac36f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cf6c1169aa1379edd85737d0d4ac21875471679 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df582b2720bdcb16d660a8f4c8c1aa7bca0bf76d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a63513426992fc2a5a1a3004fab4fdea268a84e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21003af5044b9535e107b4fa357c880dab55a8b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d305ca94e852fcc68c1541a2f0b31174c7871dd9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ccaffb9dce404179a4660422a10a0dc9ab065a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..248f72c3df3a11dc885ea56bf87ba09a9b90fd0d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb817aaeba681738f393148349190a197751d3a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fdbee03bfff14a701eeebab940dcdebee55dd44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..493bdf87b557a384a093d779c2e43d736f3f793c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5054eb639c576647380db82e046d6c814c63212 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44a13a70eabbdb30b07de7844746c9af79e9bfe1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c5d029adc532558620e77d7f4f5c35205f783a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e5ece337159462750cd43a61f0f0ae0fcf8e5f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d62c7585ba2c757bd143f057ab7a6e59c92c03e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5dc67d05b6978c5515d857638f07fcd5ebc2696 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2712e9b4691d1c64c5a45713232d9cbcd63b0837 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c58957e03ccd6c85d76e1ddf32ef875bf2f4065c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..707b73564049e39a10ff7e0b87dcc128c36c1f2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7afd5a7425e7161ff8c2b3d61f9abd77658ff9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6391ee4473db316dbd5685e3e94020648a4c6c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb7014a6b5480cc8123317379fcec9bda391c93f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74c5fc18df2caa5da286ec066052a1b73906c4fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24631ff318e3f6f7004a9977cd4831c06188e183 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9db4a0c2e13581833c88b96acb02bc02b3a7f65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f321b06ac63d32c971baec8863b9d91a91a73b08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc25971e4c69aba3ddc63673b8044410570fc429 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42cc39a8e78a32326c49315c3c9db5c25a23e820 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08cf8671208f5a4d3b94e8056cea9d35cc0446c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87314a572bd152b27a37cf4e9f68c73a62f3060e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c341a243e9ca66ba06137a4934fc700c53993ebc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15375f9f0756d185c588cfd92abd58811df57dea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21419b9b6dfcf1b8222d349a7acd254be0281a1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d481339372484a7e9605dc623dd51b924b79cf1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f083a0ab591d6faf45d026043bf7ca452d1643bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57e2e731820760c2aba37c5b1bea43f8dcb80ad2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7609a09f687cdf4f1791cb7973e5658739f06d31 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_mg_stem_tasks +task: global_mmlu_full_mg_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..becfe4b30db1ca6801bbb9a6c52f87d9c309d079 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3765002b8210565c124d39a09a3477447506cc62 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f023ccd4d3d9ad1348b8ee1737c86ce167f05e5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2993999d5a53f05e18e4be854c66eec9cbbcff1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd430a0a171bbd9237ea960f8dc7b8c2e3c90b75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1b16e86512c0bc29969cc09f978d3888f4214d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab471f42cab9f636aa71bf8e238b9b4e8ac7623f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f598830eda5f37fff7080609b667950753e971ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..330f1f52aa90c715a3779255331265974ab6fac6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..694118d119b59f7270435d0a6ac0fbc258dca415 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb6df92ac0f172469c43cda0fdbcb18cc7a2d650 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1de72b6b887ae1e9f081b13c7c00341d122cc2fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f922e1624b84e807143f7fc1bbb142710f338c93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c829b89dc10281bb38ac778493bfc57a156dddb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..362b4dbd13e1db423fb30a11b60720c10f37473f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0638cdb6d4535572c8a34a1c68ac25ea1c01e42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ead541a433df13870d7f045b2564b4f0aa95f3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_mg_social_sciences_tasks +task: global_mmlu_full_mg_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ca09027f1cb3dc22cc1aa2bf8bebfa45a144276 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_mg_other_tasks +task: global_mmlu_full_mg_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bb64d70115db2e1f79637c8f0928b615cfca71c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/global_mmlu_full_mg_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _mg_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_mg_humanities_tasks +task: global_mmlu_full_mg_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/mg/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a322dad5b8b2c05f3bc3b77ab3e1e58a31d1407e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ms +task: + - global_mmlu_full_ms_stem + - global_mmlu_full_ms_other + - global_mmlu_full_ms_social_sciences + - global_mmlu_full_ms_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0641187b82aea23875e576e47e43ac55f0d7c2e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ms_humanities +task: + - global_mmlu_full_ms_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d14420c01090b328a4f6117a62746e659868ec1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ms_other +task: + - global_mmlu_full_ms_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3db339d79e64508a357a3de0dc62d9d5ae675a98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ms_social_sciences +task: + - global_mmlu_full_ms_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68908e16ac2d25395f72b6a24da64a39eedb0f0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_global_mmlu_full_ms_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ms_stem +task: + - global_mmlu_full_ms_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_ms_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_ms_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba75026494233a4fde29e56c4de24b5d45669bc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/_ms_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ms +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec791f2a08cca1bbcbe9d4f294e42aa93334a98f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35038beaff444dc1241589a79a1f1ed2b0f53557 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79fdcbdd0c9a02a1c11614a9785598c73a22b566 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffd6195a9817669d3e1a76b4eaba3ec7af2de5bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c69b82eb5f4b243e2b83d4257dcc0182a5d57bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5821947946dc2f3bb6520331b5cb251a0fb46073 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35514b83405b51cb2870c012a443c2a11ed47983 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e242b8bc260cabeedc1cf857377e26e935e441e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07e107995767d4d7fd96ab4c53b08f40ca03d95d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82822217f4782d36a8d10afdb9b08dd856a05978 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be20fa6cfec6f4c913270e96882861cdc93967d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e886b501e59606923b487b90db6c8d74dc9b340 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a2fb6da5343d528d5a2262cf4307a662e073dc1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efdffabf9e3ba98d2b110487cab97fab9c757cd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80eba2e4795a06c026d2a54644eb5926dd9afcdd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e6caf2688b9efd8012918b8a1959c0f2974b52b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5914766232a2c7518e9517a3316d7fab569a752b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ac76cadf16c8d7feb8037ed42a50df02f7a2312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6be8ccfe1a25131ec4199bf16f735135ab566c52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f01c29b671350403a193361f4cf6c90a717b6451 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b18e8cf85be18f2ac7980c628af29b55b1554fd1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdb41802d75ffd0c1ef875a31c04da74112ad7f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4e44a60ef7a3d4f7ef7cae359c7f95934ab6286 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ebbfe6f4c20ed749b76b814b799926dddab4c46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f28f9a5ddbf9dde465b786c262aeb750f97dcb52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50a2552d0b7ee8f87cea9d4fceb9a8995505885d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6747cd9da0599822287dcc572764f8ccddb2c38a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aef3fee828e0a9f4944d9564df99a9d7281fb083 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e8641e97d5b1d62c26a0b8d1afb0aaccf132a4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4aa7ba0057274449f3250892ba8e0b11a0ca711d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6d1faabf2ff4fb989a09da310a472c66b0613fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4caf7e54727487e23fdcb6b7b5067ffe6cf59545 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b2b5c5f4650a5ef51a387707bbaf3a40cfe7d01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ddef17a710db537488125b00ae751a99cb3494a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61795f58976509acf5212696539476163145b7b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2e96706b7cd686ca2048dd36c44e2978a90a4f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d142bde66593b7b106258a37534c3a678e7cf1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..947240560206e5e186079c1f0e3f8a8c5c53193f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ms_stem_tasks +task: global_mmlu_full_ms_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ca04a13f3db9857da2dec99074052cccc5547f6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec0e44626cbb32df8bd06783583d5a88285268bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f2b1eec7f0d0687e5bb730599968eed83864341 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65da952e266b53e4c67a9c9d9881026337dd767a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..399035f2c1567c14de08de33d331bb7cb62b03be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bc74baa48b1b199a3cdfdf4ccff69fdd9246f3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..300de6771aad1aae4994c08dc77ffeb78c3fc1d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f6eceaee97e08e1620a305fc0c2c8eb6cfcc8db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c624fec5154688df3bfb017b79f48838e1c046f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a06e7f922796015310ea788ccd805a4f7ebbafa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d9a58b7b8f81b127803dbb0eb81ad41dceb5d50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f51baec9dbfbc1e5899f04f627d663ee1d88af4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..651cb72d4eb37961ebaca6a47ede956b727b596c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aeb7efa4cd54a0267741924b853025065c8216e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecbf57059919c16149449d97829f8a281acb5d61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbdd5e256571de2874c574db059771d6ecc40da6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ms_other_tasks +task: global_mmlu_full_ms_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32b350294c8410005952d248c61d9a84375d8c21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e9c77ce6c1098ac620f41e5f49ae8b1bfe36b03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ne +task: + - global_mmlu_full_ne_stem + - global_mmlu_full_ne_other + - global_mmlu_full_ne_social_sciences + - global_mmlu_full_ne_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fef749dbfa7cd9fc848e010ef5c318b226b6f194 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ne_humanities +task: + - global_mmlu_full_ne_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d3dfbd466c3b8aefdbb14d324d0de1b3653cb34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ne_other +task: + - global_mmlu_full_ne_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1f09f00211510df0ac89d1f8cf28baa0eedf214 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ne_social_sciences +task: + - global_mmlu_full_ne_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eebc1cac3767ca67a10edb6c38fd70b5c54dc15a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_global_mmlu_full_ne_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ne_stem +task: + - global_mmlu_full_ne_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_ne_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_ne_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..25f8daecd02bbb0414f665fce6ccda58f1cb0baf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/_ne_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ne +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48bf7bb188c221ebffcb2a729e5fc86649b7f05b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f66f8ec7803650ac5cff6a60dfc9bb9f8388fbe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a02aaf30cd78e9cbb9d24aaf5f3787f8a6749a21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d87f5b985c6e8a7f91b0b6646a72d290e72a1112 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f27eb4e2d358ba86a0d7567f2affc416143989bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d26edef8af4d7dc876a808a278ef2fe63d4ed0ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88b8bd86f4f4345ca2a250b3f12def25dd018aaf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51909ffc840297e9f9028e0086fc159eea82181e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40b9cb7963041a34533068f20110e20b575049cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81f81f84437cc9c0dcf847b08d2969a9099554da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09798c096fa17206a0a96b9765ce2b4d05a74684 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49d89dd43577cf8931b0209d826e07fbda631c0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94bfec4a90dfffb4a44a71cc28061587bb1eb2ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81d6ed98e045976c34e0043e4b3c3026e958d695 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73ad1a34a39b3944ef2634c780527caa3a82d471 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8c0436a44f598b83d8935e28ef61e8708415cdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..405661c7e2af99823f8352f098a24755c610e9a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b53361381a6c63cf3eb4d3a0bdace7c7a29a10c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de862b6631b9749b0f6f343c99f35e7f8d2b6fc4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c205af001d210fdc47806f322b6771ca24f900c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b624af8bf29f0966c7d9d5e816c3fffb704f743 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96036dae3809fbdfe6187de08a48d666e96245c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f49c8a5e84786200acbb4f917b8a697f936aa1a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17b28cd32c70f580fe1d6e80aa8dc82d6a4c6875 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4db01bc67184c4ece2b11d64a38f8d3ddc8b441 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6c76948b5866e83a1070fb091a3d2fd2df8b5ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..983e13cde755f5c09c4c6338bf463a1ffd89892a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80739c2c70a5aceb71209a0851f53cc7ce3fc500 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ny +task: + - global_mmlu_full_ny_stem + - global_mmlu_full_ny_other + - global_mmlu_full_ny_social_sciences + - global_mmlu_full_ny_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89e7618fbea41e79405b24889b788bf59d268fc4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ny_humanities +task: + - global_mmlu_full_ny_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_ny_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_ny_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..069a9446ec733541d3295268f7f6a943f13435a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_ny_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ny +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e3d7c33b7cd893564177664390bc62554a38e08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afbcb482fb44bac4274140232adcb014dac97fd8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f8981bd0280c4743b0461ee2b0d8587c8bc0b01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff44dd679f7e478f3663c7c8ad0aa1b37afdedef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da5ce37080d7665fe15fcfc067f40dfdab2a37b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d62bce830bd1a21fe0d05839cffc53d17f938061 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48cd98d55bd649d2b3a94587b17ab59ad9d6f71f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed77ba9c18df83ce3a87f9e6d1aca93974d6f35e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cd8aa2f43fd9f29c99d1afc2b722f613f801596 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66d5dc27b76d8d0e670244fa3d1ddf5003726b9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a9dae6225be59b56d6ae32e516846f574bcdd9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d160ffc658171e9aff349f431ba3c3270ecdbf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88af709a618435ea07ae6acc136471a593bc7e91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d835f1e003fb6aabba6b787c322667351924a6f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..558ffd0bac03a9fab192fd1e7c581841acf8c4e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce0df19215c36e911fbe4b8757851da6dc49029 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ce027a55f9e74dda90a9028259213afa97717c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a729008d55fc6da1ac8f681616cb881a13062a66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79771bfb651ac6bce183fea43fac60366d28530d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6889806f2bf0e1cac4de62afec43278bbb9caad3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29e6e4a5a3cf39c64c0bda8413c91111d1238416 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..447db75fa9d2f3cf4401e576f824eab474235606 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e543cf7607c15f25b03b4bbf1f2e78b1a5842d02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61c49e750f738e195c4152b3d3d89995ed8139ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db228d02198cf9967dc51d9bb570eb70422f4924 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62d87c862e8a57c682b3af042f56af6219371a05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54c15d66e5659537fdd0aba7eed741dcd5fd538e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f7d8b5a1c730bdb1131db0e92a078061e0112c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f53235b878aec25fb7ed3608f8cb2c2ee55852f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d413b985b68b658efc8d4771f9771a153895dd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4adf2e8b12c8d663e7941079d718ca04d7bd0ca1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9660b7b1308547c83102e1e7a9f6a603c4889f7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11a6f2d4f660c23029df81b57a7914f193b86098 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a46ff6a9c906e7538795d0494bb7368f2708315 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4606df53bddfad169bc678e41cc6e97447198cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6edade038f8ddbe8bb8dfc37b9be4ffdd620cca4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..765b2201444e56f44f6447627e8edd57dde558d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a699a70dbada19b4a2c32c8fc4d554c34a23b4eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..596d69375d64438f70093658783ae167ca236d0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fae66a7c5ecef3de82fef9fb280c3a3c2c3ca14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8555e173ab35f1179371bfd767cb28888651cb34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b64f4d9da4eddbfaa56c9318b1aa68ba17deb859 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c73f9f1af706ecde712b11fc5ddcd4792a2c919a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..456f4cb63edfc56346a4077b320ad77ff9c882cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0e0e05edddd27e5260b4890837d380c2b1206a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d65c6be1e2469f956e658ee2e1ea4caa58ee09df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c152c80efd12264f17a001672b23329a0ae85379 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5e2c7b7c2bb6d95a5628da15188429bacb5df42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cacd5df735939ab579af56ac364e8c8f37f9a272 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffdd86d2222fc24e7c888c917a0025aeab952c0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e6b5ab89b3178ee8fa98f681df7a9005173e903 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f894fdd77c764d6894740d104bc340792bcfab97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d2d0cd4d9f6acf7b8fe7b18d7aa36fe8dc75f14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a72a237de4e1c8db883d5ac37e5d9d6e61bcf522 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ny_social_sciences_tasks +task: global_mmlu_full_ny_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eeb7cf03856f104725d6dc1cebfcf0702d90dd7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ny_other_tasks +task: global_mmlu_full_ny_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1c243c8305c2680a0d1eb0cadba5deb45dbfd6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ny_humanities_tasks +task: global_mmlu_full_ny_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..deb51e119b8d12f1ad7e1c9355540c00428306eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_pl +task: + - global_mmlu_full_pl_stem + - global_mmlu_full_pl_other + - global_mmlu_full_pl_social_sciences + - global_mmlu_full_pl_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b5f7aa4584b551d6d6da59020c38422490c0b35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pl_humanities +task: + - global_mmlu_full_pl_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..241dbc1cecdd7092d7f372559beb34e6e7019d1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pl_other +task: + - global_mmlu_full_pl_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a50a3152c08a75ba85d72b4b0b41e28500b1f0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pl_social_sciences +task: + - global_mmlu_full_pl_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d11c89f119e921cc4e2301edc46f489c88afa38 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_global_mmlu_full_pl_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pl_stem +task: + - global_mmlu_full_pl_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_pl_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_pl_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..af8809dc2792042ed3144271bbd88c89acc8211b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/_pl_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: pl +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37f611a1b7804755d15290516e3321155dcc655c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c274bce178e9f12885854dedc2246993e9d44156 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99220f0ddd1d36dc346d8b778ddc333fd1f36f0d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..105926680ba0e802b515c0a5dcc096bffa4b51cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29a4fadc08d8af9e9471153fd98e3abef56fb5f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce1671cedb3a4163e6549ff2076f6493688a0f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c6353082037914354e4ba4544d2960691d813d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb630140eb6a8601990708831e3ab712f35ea7dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b42f767eae21b00657ec6c1a6823b9c9e8a2677 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43bea9764df9264452f37bb547319b1bc7ebc918 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c9ea60146bd95eae494eecd34ee5f0c72f9ad47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..365b60a3e397e58268dd400817ba860da723fdee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b9437e3fac8256e78d438cde65a037c9d9bfbf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..648f24c422edd8433c120f37bea33493020ff8e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..196de258dee0d6d30f75f5d06575bf45418382b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8646b6a393c825b2612389ebc55d102694a1f953 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d13d283c5f5556ceebbe8ea9e0e396a6155d93a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15bb640b93e0c3d28938631496197aa10c879161 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba96402838e5c30df22e46d46668f63633aab4e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f142dd82f39e5d474d9bd1fa8d647fd1e2201f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99b3b9da11cc8bc221d2a60b10bdb89827e8c874 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e99b2fb93d648bcdd1977927e6eba407dfaf554c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc6113f78f15d8d51b79bafa64ea3b0a42154c41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b61f1f17fccd40d430c7292e3337862cba73b76c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5329e1330ba57dfdc3799bf5459eca6b00484d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acf874dbf9630ff7b6c8118696cebbea93ea970b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d754904c4f04632e5a869e4a3478970ff9e0879a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef719be1f48aab6516e2de1462b411d0eda8dddb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9084c13da02ba1c7cf3a203404497bbcddfb3c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..036d0f4c0a544f0abdbc7bf5f2611d7afb20cda9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c04bf5a4da9fa65ffebca5c3d042cd0412e841b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pt_social_sciences +task: + - global_mmlu_full_pt_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc3d36104348d3a60a420360c5c47df11d108c7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pt_stem +task: + - global_mmlu_full_pt_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_pt_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_pt_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..66ba2417d166085f8b44cc257231a1e548166528 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_pt_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: pt +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9efd8170b1db9b7348ce46bc50033e0b5544805 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45390503d17e8403871c498f4e33e852aa3019eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f18ef2d8549b1ddcc64a0b3a9c18d94eb1116ba7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cf0a61b55d9c7283755b4d7c1f61c66bbf2a95a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31d7f6afda5cfafd2531f8eda65a13c676f498de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46ec823241d6af4e85b37df2d4649260d4de5c66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cf6402dd6ec95592e34932a35ef8b67829a61d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0953a105a72a03a2f2109ad7bb47eba29995c26a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67c29915c614a1d01661a72df1a56e27f6a54f6e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a6ba82e29b272d00fffd2374659d010ae8b0f9a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d66a664c39f1b46ab6e77ae665e57236c334e75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..683d6dddcd054ffc1219e6db310714d70c74b04e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4396542a55f7f433bcc361a7dd27f3ef21e365e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89fefd1c32a73e2beb44d383632ea1e50b8ec63f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea323d8a64f0701fb95f14b306a175af2227bb13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f8f008235f7fdb1a851babb6935645144d24fcd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef7a31683e7bd64839b1e67a47ab395fecd2507 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69c297805e5137c32371408199f7aed427ce273 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3fa920d7246a6e88c656bc3bce9270809f9d7ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b7ca2f4e826b4b3116b7d4f21e8f4de04e73d33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4713674dcbb43d0460556eb63e91c0ffd1bed583 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6475e995c6ef42e7dea9e14262ccbc8cdd6f0b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eaed31adcac907953964d9cbfb46f563030a4a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d09e1eb97940084b3f35e4a212f9492c68f20143 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d8c1447faca7b6a41e644ef705c0ffa7b4acd88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a883b438ef06e82e986f53d1e91c71757368c312 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ea1454e35beec1ba65d7235b7d984768b45e09d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34033c55b88be89f49997de2820f88eb689d2e2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf961c33a549b15099087ec627745a5e273f11a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5247fc9bd030a7c43b08c3146ef05738449bf953 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07e78da50b5eae4d39f36a9e39560826618f950f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2451399b8c66c2394729a43dbab2525b3d4d501 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c577eac708a196c8015b8c0413f884a275b8e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a344b1c32eaa834f8dd4f84d08914cf71c8ea099 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eeff36b979929dd011e94c56c78d418d9ef9da55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27985380f5b9d2e9aed8b8c67965c0893d78c26b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2fa1da1c1838c94b0e4a3a12a75928d0e020514 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e83d186e2f9ee63dbce9e5559618f77443068d42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3529a15c3b0e6f06b2cc7c037965b0232b03111b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e51eefe02a33c8252f308a6cca66b7a17365af9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec0826b95c63da87f3834a0f9dad7c744895e291 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..324dfe69ed57361ae56f48f16c72a431e86bc503 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..530c918efd7bf3055df23096dde71892282efafc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7a3679c9845fa23a25dac7f39a1f504efdbc88b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f4cc0066a1f5679d3529802fd5a9e561828e733 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c5884c9f668eb9816d6670260447e57bcab3dc5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb2d65360ab95ef7c128afa27d2126dbe75070aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1af8d6621226920b8319bcedde1eee2d21b45b5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ef8fcb7972b983ccdb8ca46ccb3616b0371d2dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b48f52873a61574601312edce12c244fb550dfd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_pt_social_sciences_tasks +task: global_mmlu_full_pt_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b0de75322e453c0b3049c2fc42e952c3a8ec1a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..796485864c8e686a637f68bcecf3a32136368138 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_pt_humanities_tasks +task: global_mmlu_full_pt_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e5575f04ced9e99d0785ddf1b6ff06654af239 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ro +task: + - global_mmlu_full_ro_stem + - global_mmlu_full_ro_other + - global_mmlu_full_ro_social_sciences + - global_mmlu_full_ro_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d54268b0af5ffe2cbdd10d3295aff7176b6b4819 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ro_humanities +task: + - global_mmlu_full_ro_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e58aea9a5d51491b7f23b4976124abec6de75b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ro_other +task: + - global_mmlu_full_ro_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1cb84a91129f805f5391e442c84c2ae37967ebc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ro_social_sciences +task: + - global_mmlu_full_ro_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de0e406fac5194a846713eb63cfa172003515ac0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_global_mmlu_full_ro_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ro_stem +task: + - global_mmlu_full_ro_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_ro_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_ro_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5cb6dd0ecc8d32e03797c70cd3414351e14f63d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/_ro_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ro +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c505fb8b204fc910e30e068462a1dd74111fffce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c13018cdb68e55e3f55e7deb4e8a69cad49c749 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f4caefba6f1651703a66f6ce51d5bd2768fe410 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c1387fd182fc7aa9fb098e2b9482724728fbea3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9e0dbb473cd251e83ad798d2a9698eda30fb5a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bf14ab0f5d8a36b07a06a5b6d635bae4b2c5e28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5903474436e9ea97e7fd86cb1e3988132e706fdb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bb64c2e201d7722ad353f1db87e38f14b0a3baf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d719a5ef98b6aa7be608a6e100583c75ae874aaf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9284a8f9cf6ba9c892be6251b77432f41db9fd1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d27d843707119cae640c69659f23da770ccb3ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d63556e36c5ccd350b1bf0a5419ff8a44554fca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25f30a368d61f9a6a240db0b657a262718ab43a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fa6b5d23cd039f9b3e194620cd0c1dd84f5c7c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6eb4b6e4ca6ed74a7ec8a26afc90d63ced323e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e99772e27ec8de94e5ec2773ff8147c942a37c56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be99bd00764538247bd19da55a1c12a1b99a3344 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..819937e79b6b56eb78107ae1331df479c8c73f2d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7509581cae17e6a7800017be253b4617ddfab48 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d089583f6f3afdc5bd9b07a315c6cff8f42fd2cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46d5f4723ff084e157c09a2468ab28586bb6902d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a1ae7e72571b56dac54b7f4cac108ee35a3be9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92935be5c49c374198604ef6c3c4597c6e6178bb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efd2a03fa59de3e31e0e8aaed19868f9ebb6611b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe2f97d1f167d2c37e67f85bfff4d4c5c8b9177a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0432a01f1bb3511040ece9d4305953452d5a95d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..507fab8693c7962e681e58bcf64b76d8b0da8649 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19a76707ef5e11645afa82538a84c1a57f76c2b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d27fc262eeec0f83d51323082afd596b3e86e71f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f8023bcfc7dab66b4090240a6a73d95f117b1c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acc5fc418423e59fcb7e6afe369f69e701eb6d55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ea7c933656e09cfe84d071299b74af234c08e6e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b984c55b4a5550373eed5879c85bbb0f2b0d099 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2af2cbee1baa93b7b48d9011c5ca11d55404d80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cbf3d032bf7ecd8b5ad8174a1f91ea9653b9d5d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0acaca0aa38f10f74ed13f77b9c02fd20cfefd5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c84234a0b147c90af4ce3c26c78d25eaf5183681 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09237c9ee8c619e6aaaad821688bea21e272ed1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ro_stem_tasks +task: global_mmlu_full_ro_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcb3f4853e2834a7deb38157a44524eea12292c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33b486c04d1616d6688e820155a17c250f55a5f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09c3d5e92db5790b3d2c72a9768d5b83d8532c35 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e744e1e7c49d2804a3ad927b6a6498f1b0834559 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e6d4ed7e76f0ba27550c8e43b6458ab1962623d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0e99149a67f6eaa37342782e26320ef049e5931 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850262c1e1d1d51e1f07460f6862a3980142fcf5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dd2bf54ae7daf8a3357e9c015e578c56f876dea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2ecf40d787a97a3ba35ba6ed038ec280d59d878 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db259766768e6debc757d6756cc5f5704550d9f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1e439748e9cfd8cfd63f59c01acb4ba59cf3101 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0158c5453d28aa8be0e1740a39838fd139c4f23b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdd7ca7f2495b7b17dad3faba0f894cdd3b4068b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f7f0f514d3d508c193bb2bea4012613e7fdefe0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be9b334ec12c3ccbfc6b4f0ef34ddb6589a4168e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f37228bdebc9f50909fa636d97b1a037fb5deba1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aae05dc9859b160f31c62e5e21382d310be6150c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ro_social_sciences_tasks +task: global_mmlu_full_ro_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d789c204d0d105d92a6623ac7ea513b7966629c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ro_other_tasks +task: global_mmlu_full_ro_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40ff8228470d7438213ca4580f7551b94822e40c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/global_mmlu_full_ro_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ro_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ro_humanities_tasks +task: global_mmlu_full_ro_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ro/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fc05379c0c3435e0323797030d27c7c47460457 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ru +task: + - global_mmlu_full_ru_stem + - global_mmlu_full_ru_other + - global_mmlu_full_ru_social_sciences + - global_mmlu_full_ru_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55422b437f69bbd56e9e6f0e246da26a1de3c10c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ru_humanities +task: + - global_mmlu_full_ru_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d47ccc605b8b6afbff1b7b470829ac1f0cd71937 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ru_other +task: + - global_mmlu_full_ru_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d48428a2b51357a076a58ff14dd7e196523ead --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ru_social_sciences +task: + - global_mmlu_full_ru_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70ae3edb7ea0587dfafe60711dd9825de283b9a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_global_mmlu_full_ru_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ru_stem +task: + - global_mmlu_full_ru_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_ru_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_ru_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b2f491bd4d73b02b679f4c4287555249498bd56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/_ru_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ru +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de158df873f4ec476e4ffc9d591a212043c3ef3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aab717e4f9841657565812844e819995e47c95a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d8d0e32e6cde61a7ac4894570d7a66f1b76ed9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2855ca36f8704568c90714b10868a5a0b3fba1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2efe082980f203413153c58337e5dc1d3012f258 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96d00deb8741140940781e0491178eefa781b467 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a5aac35c603ce659be0f414ee24a5425544ffc4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd8bf28bd66a7187914b42ee43d9fa6901147af0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2e080c3d5de2d361c8c8e183bfda24752999f67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70e8448ea1e2f067e672f6353be85ae066d530f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e6ecbcf28d5c4ba07dfa79aead47b46e82aa3e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f196351af89c409be27bd4e761bf32f1d5356b91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e623d78fd40ec7d03c05beabf433463834f4b3d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df35a1f608e4de41b4109aec60e216d5c60f4020 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82c49f89e5c68e437c3d7d8db6def0c08cd83283 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ed11c5faf895a5bff9b9e50933c14437917b7cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ebe62bf428abdfd9f252b9e1ca459f144862d66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27d6ad70067badc93a406bda2a06cd5d66d9ab55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7860e73e7424722dfcff244f2d64e4fa4547b568 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7596daa3b86087596bcacd192c4511067caeb80f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecb64d52c5a45e8d8101e962416d22c9ee282e7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92feccc59a718f6414ae7598432eadeb4f449497 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f586f5005f7c57f26ae9b1ce33fc718291fb121 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ffc85dfe226463b2c6e310848f25de9aeb34743 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da1320475996ed8fa9e4777d940be24617787db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc684975c906b9c8c9d3347672491e366bb140ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84887d18ae74fd049aee105968fe86a07fe6acda --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29ddf5bf3b34e4ed8ed6dce9b4d092e387d4f1d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0680bad6b72014110f5520c4a14405a3be9e316 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07ac341bf25ee3baf8f03f05239825e193f58dca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18e12bcd3f9aa77945f9ab99c9b09904e1237565 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c37522a22c9715dd437e61796b83d1e0ac4bd47d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd6bf329fdb61eba991cda68125a23bc7de9035 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8766c3489d8af3625e8a7ba4c1e2f8ce3080fad3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4edbb98cfc336dab92e778c0da10a9a4580bd8d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24cea6320aa1dd60e9bff297b89b868fe8ca41a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3160fadc3521f556371ae8aa93a9f11a6758fa63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8e480e68776c0581bee352a1efe3607c33768da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ru_stem_tasks +task: global_mmlu_full_ru_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a7b77a14d2ce5f56881970c134f9e6c7c7dc0fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c71a4f2929b3a5ac670bf2df0683e81594055a0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac34ba2035918081ac852af72b41257c4a11617d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6049ccb1ff315ef186c707530bfc999cd265b56c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d974ccfaa60f480e0fe9c9ba09427994c751d3d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f05f7de9a9ae98daef400ebe3e72fe645a839266 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59cc8deed822b5d1c6e6b5a494adc5ce97179562 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb78b1f758f52c7597313000d322ed433043fc3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..685bb2a4da295c53d7a1a37c09181355b655409e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35c21255fea4db12ceec3c69e560b09ff0bfb4ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce70d0067ab770203c1cbcdccf68a250dea4711f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce88d1dd17011bee9b15e4a1e1bf1272157225d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39fc895382665a7a4173d70d62de936c0b88a1b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dfd71cc390f8120749c4a1ab99f006b05095eb6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd08ea345de134bbd88a0ac9e366a6d1c9a33a00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef616ee16a63c4c90989ab2b394164fde267389e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8244e6562ca0668c1f24e962dc6112ab5e2febd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ru_social_sciences_tasks +task: global_mmlu_full_ru_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f4df8105ee4b6c480ee01eb0bc599e6177d59d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ru_other_tasks +task: global_mmlu_full_ru_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06f71986d3c1113ea2ebd9d6e8bc97bc442d9506 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/global_mmlu_full_ru_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ru_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ru_humanities_tasks +task: global_mmlu_full_ru_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ru/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6503e26786fd09d671993b26a5e14317d32908e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_si +task: + - global_mmlu_full_si_stem + - global_mmlu_full_si_other + - global_mmlu_full_si_social_sciences + - global_mmlu_full_si_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b97994d0495b68b8d693e84d58e5c87d97799cbe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_humanities +task: + - global_mmlu_full_si_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7600ca45413d90b9b4fab5fbe19118baf197e16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_other +task: + - global_mmlu_full_si_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e2351a28a55ce2630d7f671ca3ce2f24c083168 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_social_sciences +task: + - global_mmlu_full_si_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8878bf808bcdc726c1f0fdc7a42997a142788c5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_global_mmlu_full_si_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_si_stem +task: + - global_mmlu_full_si_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_si_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_si_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c775b20106a38ce2b9a42c91113dce664318e2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/_si_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: si +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b81c580338b6b4ac0da7547f4f6848bcb3c048bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32315245625bea5e6b01201d100c9c62847553a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7ab9539e670ad3b93b53034758bcf295b4be2fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8281fc42f5a688993b6ba454ce8043fb9a39874c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a7f5cf5b5e051f092dcb4c2595d177c4c24e832 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e54148da1d227907620fa8cbf7f46beeb5d3f3f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b797ac60370a3efcaf6ca62b8a612e5a940d4771 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba69de35d301b43405008fcfeb04d180485ff27e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65ed942495ea5621c79a905d9f3a31713d5f2c97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1418aa0dbab50f789977fb828d2b407d2bc01201 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb32cd4fda7a0bc9c2996f8c648a3f4cb23fc55f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce5ab9b7f949470dd28358809aa88c0e3048bf05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2ab5718f2d7ca0972c6a5d46577f593df036344 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e764903b503c384c864b1c5b528598446274ea3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99679bb0ab3c3b1ae4007ba0ec37b3584a292396 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..553bc9bb7afa1e039e13dd460e20202527c3e0f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..112814b6c669a598fbaea0218ea63a03bb3c1947 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..008b55377da5099f0c6ab46bb4b4cea774548c32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fecd995a07e95406f4fb29070e1d385bea85163e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d3018b606e900b786d855bc8d486894802aac4c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e80a1f2c4c946b23dadd292e73cb37761b2555e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10e15738479a5be1d7020b3d6a3be005d5eb3ad4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d90b97f710ef2d2691931b2670792d431b101e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d285c2c6a813c62ee53f4c5df13d315b099ee1ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c85f2dff989cde83531f7ad470651f094e209f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b292fa50a63c5e02b6bd9d73d04e9f6fdce4b90c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ada74f5f2d54e62798ee77e5ffa8b5dfc094da43 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84bbda2898f8832fb3e14292abf78f4717cc82ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c378798ca925fa44693a0c5340347913676b9c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13758f22fb986c3e3db008f711ad5d1c2ae64af5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fe85e14ad380875216348762842e4fa0e8c6c68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8afaa392bcd373da763d49609ca14d963f8f5501 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cf69a6894d4889ab4a54b63fc20149ee8846d01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..418927d5ba316d75bdeef7cad8eaf1e5ce0313f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de0a611da103b2f45c7175a81eafaeffd41f8b93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10212173664e47f7297fecbbd2ca6904e0882a7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d31372ada4d999fa9240a0ca1f8b45ec16ddd507 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e3d0e7c87c416910e86d7bd8cf59095f408bed0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_si_stem_tasks +task: global_mmlu_full_si_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e29c9ae4d6b7f7677e049f221fde9832ed1eae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dff414a16e6a5927ffc00d9b203515d9ea60433 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6160f02b4b9fb375cecbc13e330aba0772065f61 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de1db6c981aef6da775160b3e2734106990cae92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d48cf75c9333e3472447bc7a205e61e7c061e397 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d08b811f33ab7b1f649e8ff37e2f1a614cd6147 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3163db497fd90e3229709c90d4ea44d7d340ef08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f809bdddffdf4b4274703e58fe7964cf427aa63b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..964e6ab7b1d24be8c872e4bf914dc981f1c5db9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c04e0bbc337dbe6c191bf4a8c9239b2883e67c91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6542f14e34c0bb181ee423984cab769009b05a7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..384489798648bbcd476d3389f266552f343d50b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80f36885b740754b3daf6ab884473713200c62ff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ac5169ed493c36faeeee1bc54734869491fd6a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21423506a8ec993e4f99773332251bc3d386906f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c86ee0a346dee2d26edcc0eed25c164e5a355c32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28c238e65078413dfcfdd1fe629b177772414c7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_si_social_sciences_tasks +task: global_mmlu_full_si_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a193546072886aab9584e583a2a8caa6514b3952 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_si_other_tasks +task: global_mmlu_full_si_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..424c23c284c7df240f9e50ce6b95573c627b87a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/global_mmlu_full_si_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _si_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_si_humanities_tasks +task: global_mmlu_full_si_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/si/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd084b3284cdac35d5daf20e898e890d1db229c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sn +task: + - global_mmlu_full_sn_stem + - global_mmlu_full_sn_other + - global_mmlu_full_sn_social_sciences + - global_mmlu_full_sn_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69690862098b99934a3a6e85bd365b783c230219 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_humanities +task: + - global_mmlu_full_sn_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18e750b6291313947dfd819a88bf26893ed31801 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_other +task: + - global_mmlu_full_sn_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8e762154e8c4e6b978d306c29124d6f682cf4bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_social_sciences +task: + - global_mmlu_full_sn_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3136233b403abde2ee87f794edc49c97edae5ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_global_mmlu_full_sn_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sn_stem +task: + - global_mmlu_full_sn_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_sn_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_sn_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..30d50ba0c935327da78026e4056a8fb9cb0389ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/_sn_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sn +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4de495e9abfd59ec52b42a3697931f079937327 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ef227aa5f15f72002312f64a8239ae85637981b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8662ab9645bd99bf373f765af1ae1c00f4f09b96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f4741c04674ae249c9692210288c7f1182a39c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7477170ef5546c00cfbcb87c77e9bbaeb97d9db9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d0ec277caa41d916bda6ea04f25722c08e05ff0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f0c4f42643cc5538cd187f098d1108a6f143653 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8651ee192a5024abe1ded9effe39bc4a8dcb8e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1d1a98e0ff81247eacb8e1a82d035ac3fa6f1a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9ce08f307e45ee2da6b4ef44a43ae03ce9a89e3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae34a82aa856d75c5ec740c2575a39ccf10e5877 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b41c175ec6555a7277892d9acd598dba0e8e817 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aaa8a782d361a1a777ac2f1755e2fd7360f84c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8606e96c2959fa67458306f1301efc25cd7e54a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c57f7038f6319d5af8bf84817d2f26421db5b41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ed5b400c42a3ce553cdb51631041da0e67152ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55dafc2bf4a6b0bd2d70b864f0eedd141c2cd8c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b8ee96f09618fa6f2fbd3ba4aea6ecffaacacc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2597a7d75e9149fdba41fa09d8654db5dee697a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e6be4e6115788e02ce217d961420d6feca650ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..446da912c7dd1e8bcf96f3d0c0b21647050191b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd8cf61c4f32a26c8a8ec9c11c24a64a9442c3fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e178adfd6e82f6018026a2f282c42af949b8635 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ac4efda6b728e8b874ec8415a0975526ab7279a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23ca0b41766564f80dfacb764131c8d962a7d400 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bd9be1952012c77911a642bed92692728807198 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..916e14caeb648a2fc2257e1508426576f1859fa9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6a3e60c2c54c2f6d92fc2713f4053259862df94 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62a197c0eba0a301c435c9ed68e1f108c3716022 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..815cb60b16d7848b05b40c6ca08b871ec483fc26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff9f970e08d1206140a9e7726114c3a001edfa99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2dedc38d7e72ca888b3551b0ac305fe8b3cd2a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ef139307183fe54eef709b0f9740a0f9e1f5747 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a52c2ded23646b6c837a6ef9df49288e09eaa81b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..648c3dea0f54f303d76bdb2ff66216675813611d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca63c411a7199755284772a8fc6e451e168b2291 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d74a7f18eca74b91c10147d9bb3dff3f775dd9db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db272b3b4784e40004eddd0aa250559b7730353f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_sn_stem_tasks +task: global_mmlu_full_sn_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db3bee4d44a962c6f5eac646b936d8d47c2d0947 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a700c4e7c59ce9f47749d0ba5b2c6fc8b66d44ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b826b187e34201af1e719e556062f2cd1f99a466 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea895aa9bb6a543368b589a320c44d018b4f59a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b641f6b2afbb637d3fc41280003b62c196df6405 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2951a953c7a42b7c95e7682abb9ded91456a0f69 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9816d8b480449f7be8af8c1d57ad45715e9ad447 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ea1050502fe65602c7449e16c395c33fabd927e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e941437beb64ddb49832cd3883f6feb003acbb9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..057a197dbf9033fdc09890a2848cb5414c259c0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c9fac75d480eb4c0428f9b505f12918088e02f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e727b3cd510142625bd0a44c3c399bcfb0dc5f9b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..341322d2c83c20120ed86f0dfb0ca0c28940dba5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5448baa47394fd79fd7aa472cfc1c155069da575 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..542c709af46185561d0785a4f58f0ecb484004a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2913db53bde1f7289e7b40ddb08c01790ad8a84 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad4768478d16b0fccf2e88dfce772908b7cf993c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sn_social_sciences_tasks +task: global_mmlu_full_sn_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..254fedb4f7855067d1d5c95446821f4ed89e20ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sn_other_tasks +task: global_mmlu_full_sn_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aef6dfddb867d78365e083fa0b53835b146aaf8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/global_mmlu_full_sn_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sn_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sn_humanities_tasks +task: global_mmlu_full_sn_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sn/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91e11b88ebb1db0af2687b1dc2057020855b6286 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_so +task: + - global_mmlu_full_so_stem + - global_mmlu_full_so_other + - global_mmlu_full_so_social_sciences + - global_mmlu_full_so_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff78bfaba252a3db2b0c3c028440fedf971e92b9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_humanities +task: + - global_mmlu_full_so_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eec8e66138735a6abcec79f10a5888282d25bd40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_other +task: + - global_mmlu_full_so_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d00ea1f54b691cb797244e9941dbeab2d42fd9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_social_sciences +task: + - global_mmlu_full_so_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497b9b01e352d37ad1a5fd52757130d19933810e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_global_mmlu_full_so_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_so_stem +task: + - global_mmlu_full_so_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_so_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_so_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb052a633605159b0271f62d9ed512d447daa0c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/_so_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: so +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afb5d9084c83935d297c6e120634c9fbce9705b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79f3446df384b530c631438e174bd608e077a610 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54a2faa0913b44a6896c461538f6e92c67727393 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65bc598cb2a4a5af6c008f1dc9684e8e66f5d448 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..224aa39b24795ce983e3319b3bcf3c708fa50d96 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..758d22c37d04b323028098313c84dad1a9d6f716 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35c224308b993c3330d8dc5963264be1ae2ad73c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86428ae8238ec533e73bd0fb73cdd48c527a744d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9957a2349fba24fb2c0fc9eec0d4fedf687dbf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f51a1b123d8a47d5246cd8d3f8d72f6ddc47320a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43388d6c8d35e62defc815b972bd4ad17e2ca6c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a5563308f90a715c3216218691a957cad65fd7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97dfa147b60925a7f249f774cc40c7185bd7be28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9792659fac1ee2ff448715a5491de7204efecb93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ed44e41f83996cb93084fbc4d3ba68cb73e34f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76628481f9bc0a4ddbc18018f0d57c9671c71140 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b7645c8e4e52090be2305b8c726dd197e1408e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa75e6664fc2609971850c75df2c5f0f8a0534e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3ad29d4c23a2e0940832f9876a7aa8bd014b801 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274af23bfdcec5f12fd72bab441e36ba30917325 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bce30d68a11677821ff5996f696d263e0f5eae6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc44f08d6cfc94eb23c6c771cd5634684fae266 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55479c399128600b0f78b28088ab4846c25478db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ceb5a7014e29946b3ef4ceee91a092c78d44d556 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c403ec513ec5f52a27910ff80fc079005a8da6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8089bdf87c8b494a4657c15400ab1787ab1cd59 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32cacffea668b3cd783bbc3f05d8445009dd9c3d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd2c35ac8eeae6ba49aa768df65c516914b04e78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26f2cb3c43fc9d84809dd05ce071bab3280abe8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..730075b1c1f1afc011e4ed7997cf0d4111ad51c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9702a661bdeb4535b3f793b00031f957764c688 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78a21d5d972b73e3eee5b33fbc0482506c7e04a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c95b5562a636c817a5b9e84fce5c1116fad9875c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..632778d3b276ba032f36fd553ff1d0f86c211661 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d5ab1c53452229d9af58e781357d2f206baf4cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1372a1d17cff381788856f6cf5382305a727ad0d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19a1120eb85f9e72ec856ed88a3eb9389de21088 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1e13ddade10f3e6b90630eefffe85e3f5c8a506 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_so_stem_tasks +task: global_mmlu_full_so_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e325205640a7355d66d25b7aeb3f0bbe7d5eabe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b1c002fe5a8a6a05ea9970e752fa5b9e21f1e64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0136dc694187c5bf3732c150eea327f14a80fe0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b8a33ba92fbd59ffd6ae31ab87e82c42952ccb9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1bd0011b8bc226e01e5bbd52eae40d09583f131 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60418a65f9377ac7aaa3d9056c75e996dd5c783a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aa40241a4b85d9d7420843cf34464a95d21b5c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..421a98016b83cdd1794302bb2e25f1daa5e25787 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..721bfbf22d876a9e6f17d41bb05e55782d442995 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ca0c5c9bcb3216c7b301a53ece3c53e4e47e1fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f57b5949641358183bd1f21cd5dd0277ecb36d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7d6408e291220d4e221132cb16dbf948253520c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a03de5bb99427c9b189597be049277b5614de156 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7af81e6ae2b4e2d0b78d31706cfe663b0690418 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b52ee25901816e8015d4a83d458295f675582383 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f3847e67646ebe82f894e56ff7ff882bf764c13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6017167aafd4e39a0a247494fd2c578d9da8b46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_so_social_sciences_tasks +task: global_mmlu_full_so_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dc85b324be8c45c7b59a01d1cd537a207656651 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_so_other_tasks +task: global_mmlu_full_so_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ca99e5b4eeefbea4f282e76411f7ef7c260d038 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/global_mmlu_full_so_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _so_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_so_humanities_tasks +task: global_mmlu_full_so_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/so/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e0c55a5d231c1f4cff0329843d91f1679095f53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sr +task: + - global_mmlu_full_sr_stem + - global_mmlu_full_sr_other + - global_mmlu_full_sr_social_sciences + - global_mmlu_full_sr_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..080bc545cb3f9e0d6ac97f8d7004fc763c507ae8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_humanities +task: + - global_mmlu_full_sr_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f0735eb4439896f0cc29aabcd23c57c0a90aa4f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_other +task: + - global_mmlu_full_sr_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdc29d1f608c6ace5e8c7475dd1c6c27547d5cc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_social_sciences +task: + - global_mmlu_full_sr_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c4aa636944898c99dd4fcf7e42035a978932eef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_global_mmlu_full_sr_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sr_stem +task: + - global_mmlu_full_sr_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_sr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_sr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..6af61b3b28d14235a72fa8b548da96c7f73c5a49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/_sr_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sr +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3275870e203bca8e21570f6cef1996b412c0e9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5689af7369b640521929adde0bbbdac8e64478ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d23a4383f0e30c1b700d077fe7753f4db44310c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e89f5e61be0e2f111aa5a2a490fbf18a2e570756 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5611c15c469aa2e4b0471bfa4b79893f9f88f21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e28c3034aad065087ebb8993bbffab306f9b532 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1eac952c9e44e97a7678c648c5b917502c9ba185 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1146aa117a6b8049b105f8b65a2fed6fc374fd8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bcfda2ba34dd2c8b7c7b391eec67ac3a6bcc7cfd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3beb5b26c1d4fa89b5e5b91e7f451baf016976b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f959a02fcafa5915d1e9c826c956662f2be8b393 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e8761e0d3836bcda9a1f1f4f0fd5ca828ff775d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9325f6de74b826969db631ca40120e821f7cf1a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc4a5bcc3cc759d0df3339a9a558ccf31cf12553 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3a5a78b53d9cd2918383a6e8385ce8c6dc30504 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50f60166c482fd5721e5ef8002f5be8d2b40f5d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bdd854f1ab22207209541dadb87085cbcf9f0c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88862d212b1969fc4f02a0bc875b705db9f6e14f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f2b2952b6706e4e29f6367da93081db27338cf2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b89deb12e19d7581fc914e84be334b90a666985 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55fd7e8e662ef66d2892dd16ab1372ae9f47028d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..946acf0ee2a3b5defe39e075158bcb7326ccfb7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0705897153d5d0b16df90781ca0eb75a2ed2cc78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9721c9ba99fbcf90e560beba130c4c7ed55e234 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fedea95a6bac0ee70fb0f2f76eddf37af23c8ef1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dca9e1402fe5a89d4363f955db72b8a2efcc1ed7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b01276f650319b6ea97b1d63b9c6d46e931e0a64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f549f8ac2d25f066495e773c2c28b46621cc7fd6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b31eeeda41b4d6bff48d23940b452887f45d59 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d0f0e5c76c2e0b8c013e2e2d648a845dd3dbfe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98c401001b4dc83725084429c694cd5dddddaf8e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76e6b45c8624d95c6e9eb80b17d4ddeff2ac2bc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0ff1d95afdecd3c47ab8f84bd771631141d5874 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a30099c741f49d3d5fa3ea1897e9e9e4999aae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aea0826f6db54bce4a14354baa668c1097c7853 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..debe604fed0588bcdff83d7d1b40b4763cf01dac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..407417f3ea3e2583e01150b5d730c8128cf990c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..513a7f8775ab9ef422e32782169ff829d32c9950 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_sr_stem_tasks +task: global_mmlu_full_sr_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fca9de044cfa215d365d6edf7a18c0e5d41dc192 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8267563e63768d3d500c19a6741057e238cd0ef9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ba860f2daba6111a78a08d811766891c985a913 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecdbcea91d15ecb8612d53870fe16ae82c486834 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54bf349162b1417c8f0a09981a719fd09c23bbf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2eab8d4a06dad83bcb05acc978f8ea6367f122ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83e1b84c46f4aee1eee31061b08b58aa8770a1b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..654ee86b925b233e1ef284aa834f6a1968cb29fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a2f944b7bba7f8d342a3a1bd20601fd2d5034e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..648ae0cb03e7f6a77aa29767a784a748b3d7b738 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ee8a831bcf754d7d0f5dd005c5dec2f08761017 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b14211541c392ed442de13c5ff78c467f44dcfe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..19e2dc544db339c6ce6224aa2062f4f5a4464930 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..043024c0dcd90635962ee2b4f03d360c80440932 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24720925a41855ebe53c1d25abb4f510d322722b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc93c5e07a46d3a54c9575ac7d51e618b8165e9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b338dd6b4034b6f47ac59e2dab09215f28161dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sr_social_sciences_tasks +task: global_mmlu_full_sr_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b07588ad15476540ecbbd018e013fe3e7de1e969 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sr_other_tasks +task: global_mmlu_full_sr_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f78403edfd7b801675daf8ee10225adc82a114c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/global_mmlu_full_sr_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sr_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sr_humanities_tasks +task: global_mmlu_full_sr_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sr/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..078de8b63a3274de6e00b625f3610f8167ad63dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_sv +task: + - global_mmlu_full_sv_stem + - global_mmlu_full_sv_other + - global_mmlu_full_sv_social_sciences + - global_mmlu_full_sv_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b4628f754f5a8ba65b147882160704072bb562 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_humanities +task: + - global_mmlu_full_sv_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b29ca130b28e39600d70f62ecf40cdc479ca6ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_other +task: + - global_mmlu_full_sv_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c4a813e8ceaf1782775bee7eff3e770ce07cad0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_social_sciences +task: + - global_mmlu_full_sv_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6fd88f1efeea969d2dbd28d1aa842a92a7b413e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_global_mmlu_full_sv_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_sv_stem +task: + - global_mmlu_full_sv_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_sv_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_sv_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b9fdea988da74442edba95ebc9f78226588f203 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/_sv_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: sv +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8329302f55bfc6f86e6db0360698baf191cb3235 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac9fa560ea78ab74f23107bef64503d6b5d26c05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a88871b4efae57ab7ca198ee0d23510a4cae5c07 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2462c179f1bd149b94a45205266924741e65197 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae3fecd44f725570ec275f5cff43bfbd8494bc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71f613d4a7d5fe72ba2e740a8f7e4111971b076e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46f4c6eaa2efdcc411b57317e6e4aa72c8003966 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_sv_other_tasks +task: global_mmlu_full_sv_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06906bfd32e6e785c173305d13c272fa07c4fbce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6a752f0ae863adf73b2f0dc2b4135af73e17d3a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74086a15a5be1c07a06732ac8a835980b9b3e0d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1f4847c2f7551b9bea3422bc33dd68e67af02a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sv/global_mmlu_full_sv_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sv_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_sv_stem_tasks +task: global_mmlu_full_sv_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e49866a9fd8095d432a0ecf8e76076b1a3d100e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_sw_stem_tasks +task: global_mmlu_full_sw_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7352ad72e045f64481b102b5a66f3ce1b103992c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..602d71ffa9f91315b41dfa7692e012a36d32fd8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6781f2d521f0dc319f7787e98af1298f574b8bf1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b337d0ab8f133bea427f4085e0d413e01e6eabef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eabd5a916f1858cd7c2c6c76fe5acd7c0cc34504 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96edac9941a503dd29255f3d57047bc630b4a142 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db94a2ff4287a908026e9bccf17273e5b2b8ecd5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cd19d35dc189953db225d5ace390fa65f9cba9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9434ae4c22f8213987379f93f014a1f36a41d173 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf35b9c644fa3a50292cfd5c6a998a68b277e5af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7570e28899b60f7dd0c8a29542f28cdc802e986f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54c094db40c4f586ec3f11cba73f466954a47e46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8d5a42c00d3bc9084dac75ab79a932c7159e521 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79d51a58d088f82ccb5c7b58dce7e73654ddf3a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..523b1572bffef3758a6910a19c713687321255a7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_sw_social_sciences_tasks +task: global_mmlu_full_sw_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43179ff8432de7e09c640a2f9e2c238f7fd74968 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_sw_other_tasks +task: global_mmlu_full_sw_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef7b7f84ff14e8618af1580bf5c36f06198fcbb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/global_mmlu_full_sw_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _sw_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_sw_humanities_tasks +task: global_mmlu_full_sw_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/sw/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bc5e76ef4abf6f8296da1c3612505a9e036f83e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b440102c86fc00b4f8d8e3a5286a3f8392e172f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90e5618441a6f0284c7d1b86186c076528c717f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f036e60d50f8e4c2d98b032ac3082cd5e7ec9ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccdb849a875c73c2030beb2cbd0d70f0586c76ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f11e5657ba687bca3548e0f1b0e4dbae7e79f2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5022ce2981b9612ce5583b4d23fbd1a57a0a6e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd5219f073e7105a9177f562378d5bbca12070c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88dad05a2d9eacbef66427bc6e206a385da99204 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e8f37fc1dee6cdf5a64d3f52408342d8b15d4fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f05276252961898828e55ceaef16b767b1052a9f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf008a6748dfe708ba6bcf35e700a81579751db5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97169e931a07143e4d1bf31abc353f5d24db506c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3edc89662958185b6737382b6c76fc0f814c7b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c182d1d071d10898010a47da30b10f75ad40eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53b52f4d3ff79686c1018b6d8706685710316d02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f02170ff8d048cb3502e01966e6e77c3ae41f25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c77d30aa2dda080c2314cda0ffe5e081276e7faf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f388a0606812de534f7496e7803954cecdd5036 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d54d72bac17a0522dabbf2a7aff9830e275e16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..383596ff024a193f5a94346128eb9cb0e6144a53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8db56a85168086d0de6160f13880b6f88eb96e2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd471b8d2e4cf4b3d992a03680c51f20bd817f0d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58f577ed1206932c53832a5b8a3f4bc8f139a126 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..400a3805f815dcb48835dfb5b2c8fce407c26350 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..694ddc304f0632d3b5b0403ae370694503ef440b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b900af19ccfe1aa095845f5a5c52db1678c9f9fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3492e7249129132ce1f659e3631c43eec49fc7e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48a2d75a6714e41b9f0466edb89caa4c409e3d94 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e95f7ea19dd033356bac2d6385e3903c3fc71db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc44c1b0a1f1c616248f3b41da15ff2a645fddfb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7631419b702a0492a4c93b912b213ff3e879cc3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c2c7862fbb8689e3ae5cdf3675b25b70ca395d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..718cd9fad0c2a1b5d966e4a0b4650ada3c3ddf71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bb9170ce69de22eece49f97c7cf65d2a6394d51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..123555383b3e423607fb92a85dddb5fe66a28547 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_te_stem_tasks +task: global_mmlu_full_te_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f092416ffbd3bf7b46a8e31ded9971e2c88de79e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15b84b4678fc5868c5e41f18ca00af67fda8dfed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f0730be97277f9e1d94caa47a5915dc8236a8c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53487f558acb7e033c2ef7a2e7a71d362256a10b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fca8df9b7e1b62b0a6133511912754e4f72927cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d87f6b029e595bb6a81d110b69e6432b985176f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9348a76e2949054375aee37c3804225072ecf503 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8efe8d97b27ea77899c165997d641229742ef5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b702542ea997902dc6fbb8b83257e3fd034d3da7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..045b6e1ca5ac2fe20c04eaa51d335cee15f27a1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e5fa30806275cc5c615d58537fe7470e16adf45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4ede33f00c22cf9cf0363f15d2b6f742e5ef5eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb1906d415899f46a26ecb76e36faae1d9e82615 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ac09ce0d6fcfe9ec0d38bcf616aa97532410da3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb7bc7c81a4be3d388675c964b9f21319ed1427 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e080e0823b88d4d6ce06e007dddc376f2d50a621 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..338f080950aa736aac2d7b1ff63d990dd4a467fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_te_social_sciences_tasks +task: global_mmlu_full_te_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f5e38a9eb80b76a0e48c6528aae8844dac25a04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_te_other_tasks +task: global_mmlu_full_te_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4da26e3e17e0370a9f15543ebcbc31bca0384e7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/global_mmlu_full_te_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _te_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_te_humanities_tasks +task: global_mmlu_full_te_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/te/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa4ae63fa9d7a1243a4c6bcf11f4384f99912087 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_tr +task: + - global_mmlu_full_tr_stem + - global_mmlu_full_tr_other + - global_mmlu_full_tr_social_sciences + - global_mmlu_full_tr_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4dade158eb2ba0b672ed66d09b205db8cb97a67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_humanities +task: + - global_mmlu_full_tr_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e80a5b9d8e30e0482db4d433181e2d3153c3b89f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_other +task: + - global_mmlu_full_tr_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56fc20e14c019a09a3da4196c37fc0d4c6ab69d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_social_sciences +task: + - global_mmlu_full_tr_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51f9bb3d90e89ec77e86d89d22e9468687e0b08f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_global_mmlu_full_tr_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_tr_stem +task: + - global_mmlu_full_tr_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_tr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_tr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e322bee67158028cfafaf5f066b2b06c5bf189d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/_tr_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: tr +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e8215730fe3086fd17da2c6d3705f3df94c18cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..444402254c31c5d20abca3a14610143a493a543f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e85390bf1b17894295e767c8f780190cb3cdc286 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b1afc9cf248c4896286603c7eaf40e7b5edf23c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdfa69e6dc4ac181408c1803a777c619f551de00 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df43a67ce3c40d244893a9f723665da467bfc762 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..622854f4d1723fe8e9acea54cb61a2ae505f055a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..902bd9c1b4f8629f687057c0a3001912c3d375d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b44d0d131520d179f9897eb08eeadce7483d6bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27540d97f0f5e8629f49353977a2b37fea566096 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbcabeed0e45164a58dea395c6c662da5f6b2b5a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..628a4fcf089c8b34623b684ea201d0e28bd9571f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6feb236f98053fe44b87c1d45976d58fda64c777 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a2a8665cf779c170519fb6a1c7aabb38f6444ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ffc6dee79e3f7acde296c2a68ee3420b915f9a7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a756d102e745702cc33e49169713a1bb242ed452 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_tr_other_tasks +task: global_mmlu_full_tr_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..077476ae73e836b89cb02508f488ee6b5479fb48 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_tr_stem_tasks +task: global_mmlu_full_tr_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a0c4d90049dbdd211e3bc6e261e573b67ca54a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a585f02d02ba4fca070b527b5b657ddf2c8a8f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/tr/global_mmlu_full_tr_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _tr_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_tr_social_sciences_tasks +task: global_mmlu_full_tr_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0542a4fe21ad3d91126287fe187e9788846c2860 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bedf7f204873c9cebdae6f6acb00f4bbfc80e03c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8eae6cdde52619df8084eca39e89209fd66bc2b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ec62db01f62cc47d12566beb06500ca56dfa9fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52f91d43bfaf7af5c95393f34dbf8468e8948f90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f510c2d1058c90734f1c5d79a6818ba7922a1f90 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b657110abc7b2754ab10ce098ba0975c9b91e56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3ac0a521fa21804d313a28dfb91ffb30bdc3bf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7a9e71860cb9b6eab9af1d0ca653a7d04883269 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a61d3ae9fa1e2f5d2542fb1c6dfb92dfb1a3b1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_yo_stem_tasks +task: global_mmlu_full_yo_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92b0b526de32184043682d990d6d612abbc1ab43 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74c17559dbf18b6afe5e0e22da74e7f6c55eef9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc2c8cbcc33ca978e2f20cd8deacd40c23cf6e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad12bde64e609adcf43a18057dd13dd95d79ffd6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e85331c907b1ee1fa21beaebdae22ada5a5df58 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a6a6fc6db60a49f2a3228fb4ade06b4f25ab0f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62d9ae7bbc09fc99ba608e4b6d8ae0b520ffe5ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de42ec7a8f8655372f5ce2efece4105f854cd0e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ad3236230f03f7395a195d5cb9a95f6dbff054 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_yo_humanities_tasks +task: global_mmlu_full_yo_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..198f227b893a3758d463ac50d867e6ee3355b76c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1956c87cdf7676dad84c956e5d32ace13c39841 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_yo_social_sciences_tasks +task: global_mmlu_full_yo_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..420b1b01c8bbe069eb3b397b29e8a3ffef620a21 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/yo/global_mmlu_full_yo_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _yo_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_yo_other_tasks +task: global_mmlu_full_yo_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926c8f88e7a0e92e43e3b96b44413205fb3e77f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_zh +task: + - global_mmlu_full_zh_stem + - global_mmlu_full_zh_other + - global_mmlu_full_zh_social_sciences + - global_mmlu_full_zh_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb347da8f853d37744910ef81a62421128e50cea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_humanities +task: + - global_mmlu_full_zh_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98d4ed5e613fee710e2f3fe210c4da2abddd7e9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_other +task: + - global_mmlu_full_zh_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..235012e6c715579dade6820e7173e6fbb69a7f47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_social_sciences +task: + - global_mmlu_full_zh_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..660486a4f41437936635eb61999a13c457bc2ab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_global_mmlu_full_zh_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_zh_stem +task: + - global_mmlu_full_zh_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_zh_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_zh_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c83d4957bae4fa07bf63d4a177331fd2954c37c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/_zh_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: zh +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42ea6276b627a8738f2133e30a509dec2f25c28f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45001d14cd72fefba40209a38ee044003b421b91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37183dc793c238e307ebf03cec66000336f50a45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90ee0ea4fe299091b204e411d9a0ef1593fc0d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_zh_other_tasks +task: global_mmlu_full_zh_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba2031feff45c3c7db26a8e30207cdff0bc81a1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..860761b4f0079f95ca4646ad8c111d1542863a9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53d01965f9d5cbbbeea2ccc9278809854b100d24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbd2e4be875c004d729554a69a02b39671df8e50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/zh/global_mmlu_full_zh_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _zh_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_zh_stem_tasks +task: global_mmlu_full_zh_college_mathematics