diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4365730e10f15fad4ba13aa3b12e154754619e4c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..371fb521c98099c63853f5ad3db32d3a0778f07e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c080b64554ca8788cecb1e6096f6f7a327f6e307 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45b07299c36bac759348bfecf2827bd7b10ed3bf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e33b5836db631fcef77d2420ff8b29761300fff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f035162d0eac7048eddb22193ec64e8e596744da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d4dd34ff212178d2ea1414d219262edc5e646c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2245b5273190a42b740086fe9cf5f19c8db8ebd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd920305786770287f7cade75fbf59e784e35295 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f38b85554507d61b58b4e51e3e413deaa4761e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ar_social_sciences_tasks +task: global_mmlu_full_ar_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3be1f8f9527fe18c267a0bd5a7c10d8506b37c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ar_other_tasks +task: global_mmlu_full_ar_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c7f01a5039162145c535d6fcf59e4af325a6402 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/global_mmlu_full_ar_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ar_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ar_humanities_tasks +task: global_mmlu_full_ar_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ar/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb5a260051f497f616a5f527763383aeaaebecfd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fb69d57fdf680da1fe6a0bc73a1d6d6194032a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..442045f98fb0decceebcf27328aa758cb2ec7e47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6849ffbb246bf3702139343745fbeded3b84752c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dc8a2c208e56fcaf8e02ccd1c111883eea81cd8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..941f635570bf80b4ea9fb11a899379da666391c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5918b08aacf3b9697ca4c44aa943aac8dfcceb71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f7d1f105662952141821e1a9cb4670dae01fbcb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b54c80db067150628546f7d3e70d9331301d2d1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..371d61cd67a3803951877a0cf2581875757b01d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f2c8731154480d59d3515812f612ec23ec06166 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0022c8242c23acbc206a58b30dc25d28eff323fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ed6c6c19f0d6cc3035dd20774540ecc072ed34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9118a11cf28bf871836cc79fdeb15ef545afdd1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7fa3c1ba62101e8566895a0824346db482d82dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..067ec0eada3fb02e48fc38578b884a50222f1bd3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12c775ab47ed75ff643f9f22bffb423ff31823a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82809b152ad80a915941562d122f029940e75adb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a14eb7031a8cbefd3450e4fa7f56947a83caa6a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a84f85fd0b711e16dd0781d9f0943b6ac19011cb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b10c59da49cd1fb9a454f684fdd656ca9b2bbf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4231ea265ecd44cfde35f33d9ea49d3777b716b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28dbddf78eaf47e21ddeea2f69eca240d4139343 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc8aec7daaace65e6d33c5a2886ba4aad7f8df67 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16a3c204aada93775badb2bbdfa1ee1e2d141d56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a25244a55d8f43f340159d8818968f1234d97634 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_bn_social_sciences_tasks +task: global_mmlu_full_bn_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b47f516de9364292137cc4aa6b3b39c8de477a14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08ab51e2ffc22baa8d76108895f6807f2a24c331 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0885a1a99d29cb8deac7947471e93dd332b7c858 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0eb0997def69249d9a729aac47557a24fb9dd08 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_bn_stem_tasks +task: global_mmlu_full_bn_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d006b41160adb4e9a642a188bd85cfe8bf7f4332 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520f9469968e8c004713e97898fd4b47b88f4d88 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88caa977d9c6624ccd1fb5a4a15f4adee9f4321f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ce31f7b1a67868c611d471753988cbca407ed29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5f59e15c36e276eb4d9bf059a4da73b0f20412f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_bn_humanities_tasks +task: global_mmlu_full_bn_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..422bba55011a3391e3bdde0c1c683f2af9b70ccf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/bn/global_mmlu_full_bn_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _bn_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_bn_other_tasks +task: global_mmlu_full_bn_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_cs_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_cs_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce2189a0c8e97ef2a85a6f5ed0471d5aa0f25da7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_cs_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: cs +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..419d5de4fb7ed778fa46654a9d1b2a4b40496ab8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_cs +task: + - global_mmlu_full_cs_stem + - global_mmlu_full_cs_other + - global_mmlu_full_cs_social_sciences + - global_mmlu_full_cs_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4b4aff39d367d6e804a7ca9fc7f3ae79f934fa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_humanities +task: + - global_mmlu_full_cs_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..302912e4507c68d36d43bc703f152f9cb3ff777e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_other +task: + - global_mmlu_full_cs_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3fed76e40b7c0ac6013ac2ca8b1153cfb227a0e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_social_sciences +task: + - global_mmlu_full_cs_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..898bb09272be41a95941512ec52e956dbd8e0704 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/_global_mmlu_full_cs_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_cs_stem +task: + - global_mmlu_full_cs_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97d7354d45dcaf5e2441d15d59bd7a7afb25427e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b5a5f993429701cbefe8640d2d7f486b0772c3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6db79c5222496823584ccd4cf6c5732a75b0483d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a17c60517c0cc77e25f809ee17cb09412143bb4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c6597b4c25121cfcd45026214c971dbeef3823a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..713af5c32b017b34a237d622f8b6e17c0c59b4a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e09563f927692fbc7eccb6d8083b61045336d070 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7b868c922fafa4d58fb08fa36ffc4ee496d38da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_cs_other_tasks +task: global_mmlu_full_cs_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0be19221b719c1f6d9885516a3e1c2972f2c9083 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/cs/global_mmlu_full_cs_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _cs_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_cs_stem_tasks +task: global_mmlu_full_cs_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d17d047ed69532738d09b035d3aa1a1d745f413 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c272287ef87e5ebd313692a32b521e90f3bd326 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11f804a622eb4ca1b2af58a70e8d1351985903c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d5b4d775aaef844a5d7b23f90a86bd0cc13056d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3f09c7fb5e7b69935886c254d00a5a44bbfe6be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34bb5918578631bc0db9656c65155761e59dacc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..585e99b3c0f4d8ca5b6c54e498e71dbf2ce22e25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd09d6ad16f2c86c148e1ac1b64ee796dbd74e98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfe82a9b63b870d19e073efed4dc2caf37775ff8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_de_stem_tasks +task: global_mmlu_full_de_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7304da38e40fbe1099e2c6e23fab8a27708722b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2143e4f17f3911ed57b255d0cf1931c100e318e6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..015498683ec50ef536cf225222d0a0bc337ae3e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c8bd533b6a3d71e7c2f591ae7bd7e9c2f745787 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f03361ae6421fe80f2d68ab547baf110533c15a2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a36519a7349840359de7c4af80429c2c87c35c37 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799065cb7528388aae0e644449c27d69779a68b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5f0372b555f9801f1f37a84449ca495de7d45a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2145e87d4ba25f03885318d52e3ffb7633a600a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ad55e9773a667c5cd11cb6e580133bb230895af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f4e338fa3466591c4972094a9aa06cb3a0adc3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a1214a6cb7b854348aac953bc7f5c562bee2d0a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2d49ec8c903956eadb5d1e50b203fcfc3dcd90c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b7d23a83fa84f1b3c559699d07c40c3a46d3da7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1a3b22ee55e66b6cc225a921548f5fb8698d64a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fefef9d016d443c389bdc7224aef26c90e245ed9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35394ab5bb0b9908aa4f6701ff04640e3775ceed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_de_social_sciences_tasks +task: global_mmlu_full_de_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0f2f59570dfae2815350009e617dfb711918192 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_de_other_tasks +task: global_mmlu_full_de_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a43e6fe07f465b8dfac892586f15029edcf76b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/global_mmlu_full_de_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _de_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_de_humanities_tasks +task: global_mmlu_full_de_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/de/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d44d8c8b9c3dc919ec9e0eca9ee0ccfac222748 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_el +task: + - global_mmlu_full_el_stem + - global_mmlu_full_el_other + - global_mmlu_full_el_social_sciences + - global_mmlu_full_el_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f07f2b5243f29dac15f46ed4ebc0536b6624bb41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_humanities +task: + - global_mmlu_full_el_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..938292f095a0d6e1af6e085f75705d839d74f751 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_other +task: + - global_mmlu_full_el_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e72e1e9c89837b812db66b62c44f272d3ca0dfca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/_global_mmlu_full_el_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_el_social_sciences +task: + - global_mmlu_full_el_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc56c0696f2b21c98c212f8b7443a6b7d2c601dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b2e0e7ff4dea18826a1298914ae8387cb75a06b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7faf7389ae5a74efa0b40aa954def2329302fd97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e8b5bb43664ce26239af40025cfa6d8bc7125e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ade42184fcf5b554bfebe8882748dfb2ca3600 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf3aa362cf68b60c48dac28af106b3e1d66fbddf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd8e1dacb6fa274f19ae106fe42ad081cb0adaec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1ea08597254464f7c12286f5f6e4bd8557fc6f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ec055b7fde1aa3efa5009d2397ffc2380f6ada2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b16b545b029d469fc6b000ede0162d73218bc78c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4630f7f82d1dbbdb33cf2470d82522c50d22fbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a40228ea3e0fce6cc9afbdaec8fd6629267a69ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7baf6e602412356ab271bc7ee9388f2673961bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48e590213417ddbf4fcd72df0ce04b000e0e9081 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..294c3c5dbd6e7a5fffb62ea28de6045d2f70fbc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4373d82e362dc56edd734fc5bbbeccf57e76f61b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81799a17c9c3c26be1704fb1109bb1eeaee706b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6317eeec7ca9aa662a8ef0fc3e75671104438a5d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa5958aa1817fbf1b1cac1ca0da985d61a711b80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38053add37dc82b7e97328c29bf2a700c5ae505a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fe73214dc389598f34506130a94c6d0c13e50fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8beb1e3f525d454b7e5f25ed50cca3d44c1f0bb9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22c083214dc41c9b6d13c0bb0a019e1f11f8d52c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3b330411733cd8c1b96cfddc296026f1c4b538a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63ba6a05449b86d16be5819086fbbc425244cb56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6ff6e2c0938a513f93f3ace33bfb69c7b401457 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e4deeebc53868dc12caffe10c57fe548d9e687f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb8757033d499b8154d8bce5cdf49467875ba9c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e82d1b5304bfbcbc0c6e48e328820b657529fec3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0003184cc21cdce0e7ff70a04114000a916d1172 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5e0a3675aa48def57616e76c64f2fe9b295cb77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac460ea8bfdbe8ec13fb74426d7c8c74eed3ad7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a40e04f2e8ce255e1cb415ba1c6cabb5d897760 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de5075bcfdb0d4959b9a4638b81c351fd95d54a3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_el_social_sciences_tasks +task: global_mmlu_full_el_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fb93f2c8eb0b60a978a88377a0cc06786e7eae4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..624e040add336046e3a7748e1f93eb4351252e92 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..180f3b25e44a21d081d8efd4bbaf9754766e0841 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_el_stem_tasks +task: global_mmlu_full_el_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40487fb17fea9b6275277f2788e7655b13a1c837 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..781d41708e306028bba373fdac5cba326530ea49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ca0114644d6becab5ae8c56fcb3b11dd0baf062 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66114367bec5d22978fa3ed8b91d7e3f0ca05a24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_el_other_tasks +task: global_mmlu_full_el_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c553ab7be81a621d20b9a71e7a0d35dc9d5466e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25b121b69039acab1bf15344f21fadf4e86dc97f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5938a174e70f3e197837162419008d38e5b0770f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b457038023c47923bc270de3e1bd469f2f80c6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/el/global_mmlu_full_el_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _el_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_el_humanities_tasks +task: global_mmlu_full_el_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_en_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_en_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae7da46b25bbd0139e8655134b9796ca0f7afabb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_en_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: en +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c82b6465ee35149f8052ca27a2adfa2e426f3e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_en +task: + - global_mmlu_full_en_stem + - global_mmlu_full_en_other + - global_mmlu_full_en_social_sciences + - global_mmlu_full_en_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4455fbcfecff47197958935d8662b8c3d432c9dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_humanities +task: + - global_mmlu_full_en_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cca60e529e6cd3363cd389128e5509748ef1cb1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_other +task: + - global_mmlu_full_en_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..becac7a59ecc3f807f1e76db06290501ed6d1ce6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_social_sciences +task: + - global_mmlu_full_en_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71aac0612bb7b92be5825adc32439426280ba7b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/_global_mmlu_full_en_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_en_stem +task: + - global_mmlu_full_en_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d7a5ed80c8278cc185e5fa4d4c32b908df44790 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2267ad88190006d76f6441bc0fe57783ee80210 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6999c30f68b75d1cdb489c6da00d1b220adc6332 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56a6e490ebbb4ca20413e5109f0ffaef83bf05b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60425fadc666754f055eed2a79d47ee69a5dbdc0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b5f2f8c5a7aa94bf280494396a03c7f2bf93516 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e2ab91f91c4b75ff5e57da7bb87844059698a98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9abf38dbdacd0678c1dda043535b0cb7cd5c7076 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da6199fb39a769afd96a339b969913f604d1628 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c568f36b774e9b77efbe6b075861768dc91d5033 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_en_other_tasks +task: global_mmlu_full_en_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac0440196d5682318c820724242d9b0f058a1ee9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be47dbde08f8701c2f9b46b226f89ce40ac887bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86180924fda604180d7d0897fd754e7119fcef6c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a75d329f4d300f849897c3742e4077e6c9296f70 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_en_social_sciences_tasks +task: global_mmlu_full_en_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2568993f2ba8edfafe179a4a3a2dc43f6f0e1a8f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_en_stem_tasks +task: global_mmlu_full_en_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..109ca44ae2622cf99b48fecbcee4ad304fef5c93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/en/global_mmlu_full_en_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _en_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_en_humanities_tasks +task: global_mmlu_full_en_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ee91f710cf4372f01de5d0ea60749707d2c8e5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c2884439ea7c9f44156c5a29cb0455e0fe04fb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..900936eb1b27ad153842e7da9115325a27fb8b2f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a654fe86568bf5c602d7be916450d54d6ad42a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a11e310c89164737d89c6281f9589c038c2635d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_es_stem_tasks +task: global_mmlu_full_es_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a31b4c268096dc79b15249de064268fec3d4d784 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22136569ebdb5755787b1bc37e7c74d94febf58c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18fc7a23e7470e44dc15fc47569c083aaecc0e6b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b3955a93e27cee34ab96ad17ab688eb22418043 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57095856fa16bed191f32241ee7b8485f7a770f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed31f8cc97d0d35c0b6e31573c095aba1b847979 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07746d096513d1508cb8b113ef945f40e518f153 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3853e1622e915eed12b6d667fc82b644fa07f49a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b75ac9df71c2b002439dbfeac3e860c06692d07e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da8fd46ffe348f81607e2cf4d7072b0955ac0102 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddd0ab3d121a4311a9eff4533460d1912a1f5239 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6be1ae817fc5f2d1127b05d31ab7b1cb3617a841 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cadc7f964caa20ddfa75e9ed057098a52203a6c0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72609ea90d020b5faf9fcea99139394e826ae330 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319123c61992678604578547f13f95249b1fc62b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dec44c2938c6d3cec700c08e6cc2a0c22d5d1fa4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a18a3942d6a759c9a31529187f8c84bef3c30833 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_es_social_sciences_tasks +task: global_mmlu_full_es_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b06431e21049838070abeb45458538d5d283d837 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_es_other_tasks +task: global_mmlu_full_es_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d9d6b792ca9592b53f60e7a3b9fe52fcee62a42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/global_mmlu_full_es_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _es_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_es_humanities_tasks +task: global_mmlu_full_es_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/es/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/_fa_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/_fa_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..952259b2fda8a1fa50cfbf36194d8444b8f8c947 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/_fa_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: fa +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7da860f408f871f6d5f55ecf7bdd92c9aa81f8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/_global_mmlu_full_fa_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fa_social_sciences +task: + - global_mmlu_full_fa_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1014795f0ffabcb8cbd664a35c4dd1107276dc3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c0dd60b702d06470110f6b94c8ef3ea7bbf31b8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7af0e21f2e84c0c38bd0839c95dd1e00f64f7b6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31ae6d71c4e6586b57c1f54fdcbd49c13644ba32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b099f413a68244140b4ae16bb5d9231f4e34555 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07491e5bf7f39a26d409cf15d4613bb7cc537acf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..774f6b970e90239efc066e80b544dae0e88edda4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13d6f5a2ac910fcef23865e4215531105932408a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e415b8cf7782d520499d84f0101f4f77c6f3c13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae47213b39e79bbee367dace28219e446e388a99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3f2ba4ce1c465cbe343c6b358c241e8140d4b18 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cf79a9229c102bd62100c56cd3d3404f7249026 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab7aa8584e93622225c370311282c518671fc21c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b83f6ddcf82ebbe572c7e76aa636332f10ff6efe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cab2effa03a184bab3d8486530151642708f6e75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93d11b75b9a4039866238ef565766e69dbfb45eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59b6869bd685cb7cabd763d03c3497437c267ddd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d15d4b4ee4336f73450e0415f949be6404050b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a02df4f13acfff661358bcfccb76fea1d0a74adf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e18b2c7b6bfe10d4a3c0bc834c5e64c1ef4725d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d94c7e89fb70e84099c6be16ddcb938e10a96e27 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e10078959c1e99b93684ed5f9cb17d58cec8d3fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9ad06337b8e6cdfd3b474d9baf2e2ced4f267a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c733b17def83597d8787fefd9a12df5ed63ba53 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f88f0aa335190798cdc01c9acd0927822c9065f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64fdef984a0f16d87645682bf15a4df59f19dddd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43a115be4261a4ac303ef88a5b0b34471b1837b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebb4e82d7390d9f8496a5b31bb4b07ada19f87fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0041e33e03b3ba765e14bc5d0e91eee76792382 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66f38f54bbbab488cb994b94e956bbd41772f1fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95a2addea9f833168e40a19cdbedf741852e96a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..475a71fdbbeb22532ef620ceda4bc70d03445948 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0d6aec2bbcf9c60bbb450ecb4e5fa4fec25a851 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d82bd5a64a031539f02338d163082f2c6bdc026 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8e89d3c7c16793541b93f5ad406a753e81059ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e4d1a8a7ad94e6ac37a4ca3530cb982e0f9bb50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_fa_stem_tasks +task: global_mmlu_full_fa_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e592bae4a02a3b7959964f49f3fc5a58e49c6b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0e7ef1f42c6ec8e42bc7bcb346af99c9f136cdf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c31679ec0ce09c4626045b59dcc0e835d3f28cc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..652d5a3335974f6e32ab58371e8b6dce74d1c54b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16adcb26284ac93d5fcc64cffad556399bd35514 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92d018f2508759a5eba64ab2a17867321e7e43f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae7e065ecafdad36c7fc421fb05146ef3a8ffee3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd8513da780e3fbc9f9ea604c7b3b4a6800e5bd9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fd6bb3d0a359bef99ff00477d3bb70c0b889bcd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99f6c316413309444d15268e4925aabed113e9b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fee460a41ec892865ec1ebc540d91fb4d219abb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13d67d457680de3c7935f0b4b4c2571378a65201 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e821145e4170de1337027c30b58aa9bc833e443 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de6cc3118d20e7ef0a8ae248466192ed03134eef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64d5fd14a5ae149470df306e0b997372ae863935 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf3d9564b1281dd85210e9a08eca428f1cd96bed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38d51936c799103492b48f79c1f9effd5d4b0cb8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_fa_social_sciences_tasks +task: global_mmlu_full_fa_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39c5188d3406d388333cd93ed08a107c78f9d5e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_fa_other_tasks +task: global_mmlu_full_fa_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44e6fc82993da5e4543092433e2b13868281b765 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/global_mmlu_full_fa_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fa_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_fa_humanities_tasks +task: global_mmlu_full_fa_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fa/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_fil_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_fil_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..32dc097a1d098439f5bc038ec4fbdba97ea6cdf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_fil_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: fil +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml new file mode 100644 index 0000000000000000000000000000000000000000..659c9d4b9a60b23818fd4b4e6617f9df92ded70d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_fil +task: + - global_mmlu_full_fil_stem + - global_mmlu_full_fil_other + - global_mmlu_full_fil_social_sciences + - global_mmlu_full_fil_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..061eb818918e2279668ecc6087e8f76de9a6698c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_humanities +task: + - global_mmlu_full_fil_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fea793eff0e83703853feb8b28d5f0f4a24dda7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_other +task: + - global_mmlu_full_fil_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9f793305951ecd337a7522f986bc5a5935f5a16 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_social_sciences +task: + - global_mmlu_full_fil_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e567c70b16de81204a0bca56062d14c4a5fe2da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/_global_mmlu_full_fil_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fil_stem +task: + - global_mmlu_full_fil_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7eef19d6d1a0c959e3d7bdf60adbc8e02f8e0a48 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e87d8d803487fb44ddd992d57b13bbdd82e54e7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c258877fca7f68475467dc32a530d0274abc841 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..139f3cccffb7e55ae7128ed615b76c561aa1fc01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc160a99f142e662840d33b82d51c867a4068ef5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff6fa3d845e14df911bcdd5ad23da3e63c7588d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61f0df50865a6b95a9237e6ff1bc8ada97660c8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1385b9345d53f99b935764d70b4a63d31bec5af3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afe15d7d57504b28c4dcf4980b0db8d9971612f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..221289f7d0ec8a5c1d31f0f12f0f688a00fe9195 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..863792b3cf27e931b4d42c0f96f1f3c68267a919 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7971c6065cc62f24e9672417f4bdb812092aa7aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77a75ccf0c4bd02d58b1d9c2a9620e9fcf5e923e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd98fc8db07bb15298c0e7c6bdd82b9c184bd408 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98e48a27cc8962d5bbafcd9a39ed9044b2a374ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eba4149c2e842f4c9902f2052d9cfe47b041d751 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1796059f72356dd890f53bfd842957e38aaa90e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96886181c13c2379a0a13d412a981158dc367e2a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93d94120d327f185b11aefc04e58e9e68391c8bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ec56d5bee1b08d299e5bb5f28acc3d921cca59e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d86aed2344f5f4e1e19231769a5ebeddf38fae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fcdec0a10b90d8fd61f9a2089f6d3f048e2dc76 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96268192265344046079f9e7abdfb9a40a02bfee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f826de3daa160c9ced3f383fda400da34c0b4930 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..104a70881b304fd59ead25e02893891851fa8359 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d499b4d948b2a988af49faade7a275450730dd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43fcc04d93cd66edeae07a40d69e0497384e5695 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..175f31eef74c214fd11b660c4cff66ae8196e128 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fc2dd5ca57af8a8c3df6d79cf809380cfbacf26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0540d57c845a41600aa31326745490af328374ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0801af2d0536d6959034c65bd6156e4cd4e088e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..724b7ce81de1764f0b0ba878fd6d93d408941cb7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c2c114123967f126ca40f7a22cf5eca97c3a743 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1672d5b2663ad38baf140903aeece3b5ad87db64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c5da91c50cc864d0068464ac52c4ad851e3a9ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea2b20b8f28d320e73a02b23af2736bf3ace1f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a30c7243f07dd46188b08085c8440724a9da418 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2a7062c232c0165020b3cdd7d9788950fb5a0a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_fil_stem_tasks +task: global_mmlu_full_fil_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ea5683556e38fc3eb03128709d99fbef2e12cd4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d4490adda68f3db5cb97bf3514d5ac7868d2d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdeb0984f5d5e8b5a18c38bf34a6bfdf67321152 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51c56a3bee09eda53a834b3f712d4ddadefa35e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53148a54249fdfd2104cf5f92357f151fb25b3c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5fecf2edf0d8d6625d9f27b33a769ff45ef51d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35859dc2da609cf6fd8e498749e747e638d3c459 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc2d414eb1c67c48da44275f4a0b189bf6ee24b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abf65fd2fcb66180ed45032df7e65960bd21473b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04ce343642b8af545083b1dc21ad269f512d2c94 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5694cf45bb2db7ae2a0a2dcf60b2d9b696842bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7fd0446f901869c9a1141455f9c3d55b56dcc37 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9ce14aa062dd5ce11253374373157c24d6160b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdae52986567539b429436b2ff5609a30856307c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a03eec195604894bf1666fbf745699a98f6734c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc0ed0523e6448b4aa339636c460741ee11d3e47 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed40afb61259b660310b1a4c7d889997754c6e34 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_fil_social_sciences_tasks +task: global_mmlu_full_fil_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85ed4d427fb75031c99277423fe9765575d56eff --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_fil_other_tasks +task: global_mmlu_full_fil_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ee6bce1834b1a20e3b1d23946e97ff1034cc5c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/global_mmlu_full_fil_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fil_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_fil_humanities_tasks +task: global_mmlu_full_fil_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fil/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_fr_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_fr_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ca79725c1378eb5cef7be5d952cf02751c9e68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_fr_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: fr +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7857a193281519c94d2b087e47ecb87d93ba7be9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_fr +task: + - global_mmlu_full_fr_stem + - global_mmlu_full_fr_other + - global_mmlu_full_fr_social_sciences + - global_mmlu_full_fr_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..697e3a29e03b0a03635f6ba49905682ddbd4698d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_humanities +task: + - global_mmlu_full_fr_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b2ada6b32d371f79cedc9539c3ea21a9bb68e79 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_other +task: + - global_mmlu_full_fr_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac7e4605614a20ccd67d8d88172436c105a8098f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_social_sciences +task: + - global_mmlu_full_fr_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c81d601f35758afefaa6b804981941f494b8196e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/_global_mmlu_full_fr_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_fr_stem +task: + - global_mmlu_full_fr_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf7d76c383a6d8087c010b86e8fe7c5d2883d6b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9a96927c00636d01609cb2564dc5d3fee228638 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e4ca5a7691cd4800a24ff7032e1fa66106bdc72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df3c1fbd6b78ff26410002dfdd525d69d0a6c876 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0daa2e6ee66095c0065e05c3f5f966cb042e9ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e997578893b0bd98fff5ebb2696e4e759a032ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c1c31897fcab74c2eefd8e564c04f5519230a7e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..078108f8c7e73ea0a2bd731648fac024b5d5a042 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf2f2940e1ac269c5676cadc3639bd720cae70fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c9ccc80e79accc520c4c865f090ea399897aa40 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01dcea37091e85c421f9c4a3009d5159f69f5ba5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..794f64be1db599f4bd21d60297397901463e6bbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12c6afc2abd8fb8728d727455f77bd7e77b49c72 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f33ddaba54867b2d651a3896b1d87562efae508 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dcb058564a29b77f441e872340594c0f8b42d56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2658ce96a477c39d7a8fa6f166b73aa0a4bf0797 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5239cb1c194a08fb6adf2b6535ae5e2fcbfb6776 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2763dcb53d53033bf7f6389f0b64d2f361def3e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a6a26c9bf4bc956d29fc0c6d7cad66e2b2cbe80 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ffacc2982af8fce14dca0d277e2795563dbf966 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d17204224942f109607411c8d47fd1d23148419f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9788e7be702d49a032954c193524636049a1ed39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e2ff22eae27b956947253cc1200bc2ffaba2bc8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aba3b6121ec8f3075e1d3873c417c8003b5a962 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21fb1df5b34f62551946632c754c03e6b86ea1ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a975d1fcc7cde59838f4e25530e5b4e075a7d455 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff654ff35a003c1c2392d78437c25bfcf28ce772 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4038c956dbeeee5668641b0ae4204d1a91bff2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a65da78026eadd2ceeae2461884bbec1156f24af --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37c75136b2ba62f3ddf2170ef5db426ebf4ddb48 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0e123f267894898a64f0cb16ca6e02f4c5e38f3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2a9cf6b143899e697e80d2f5cfd0316f6727c5c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9e9ece96b7b55d26517e842e719ef28400cab36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eac30d27f9147f22c136b90fdf54f6235b750033 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e15b0fb1064d77e6035e825a34ef13c9162e900 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42079c92eb665517e9afaeabbc016d5ae2425fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68ebdb717525e90095a2284aa9d0fc95c2018676 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25a8df3a87ed22e928ece07116306dc3b488f48b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_fr_stem_tasks +task: global_mmlu_full_fr_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73f7d869215c82216e9f3826f1313a25d2834ec0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a19b83e9a2855f0dadfcd8c5c33d962ab5733d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d15774f13f7766baa9f4a77a2e11145c75fc1b99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7519709782e1dd04c51f87a91ef32f403ae4e2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f625921e5d4ebd4136ee0e36d6725a9e856ebf45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4575ca044297eff4da62096a1f4911349c559940 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61521c91da4d5597aa1c31450e1b18a0115f9771 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f771c34254a811960b476324175d5592414c237 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bcac0f5b900f65de1b502464da1275bf7bb261c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a06a7af518877583fe9f1e1a83eac47afa9f54cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ecf2e8e7014784d6a6f644894d719b6a36f3df0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..983a2d38053e4beeb5539064de4583e3c9f59017 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59d9aa309b3c175f590251de47f138b34feea201 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d84b7ad07cee145f31c5ec81f7a3abb02abb089c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcd82b7d8d6e088d3b5aed15dbb708891bf75aa4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff7b8fd3185d377be099f72e04ba03e21d4839eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d92c20955a118004738b6a7c606dc72a0e7115ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_fr_social_sciences_tasks +task: global_mmlu_full_fr_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..211c96a79fe7cf5166cd18d10e538c9317537646 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_fr_other_tasks +task: global_mmlu_full_fr_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1f168aed5d7ba6d62ab40da70f71b00916bb620 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/global_mmlu_full_fr_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _fr_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_fr_humanities_tasks +task: global_mmlu_full_fr_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/fr/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a50084174aebe37f832c02ca23f5a2fa7c1b9f09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ha +task: + - global_mmlu_full_ha_stem + - global_mmlu_full_ha_other + - global_mmlu_full_ha_social_sciences + - global_mmlu_full_ha_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84cce38d710ab046922d3b371b68f9b939f4e248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_humanities +task: + - global_mmlu_full_ha_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a6ea0dea1b57902569321973002fbfc072453c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_other +task: + - global_mmlu_full_ha_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b520a5dc50a793f2cf5e349baf416b69cf76332 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_social_sciences +task: + - global_mmlu_full_ha_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6213d280b995bbc25881afcc989969965cce1308 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_global_mmlu_full_ha_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ha_stem +task: + - global_mmlu_full_ha_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_ha_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_ha_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8521fe50d463022e4066bec1fd24dd458c48599e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/_ha_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ha +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62ad5e7d7bec45d82452d683a6a7682b65dce287 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ead0f6c11557f972900bbc21a4a94e578e73d93 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1616398f862e1ff6b94b6ee28a60596eb5c44110 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1719b0e0d609b3af0175267e071dc9bfa09d87e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcef5e272db17444b7ccb484be0cbca2d012e181 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f28256946943960f33f8f6b4bac63f62f755284c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73a422e3d790f5e70d6990f3008f6131f518aed6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bdb65c9e0542257eeb7e804a5c33f864b2264f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47e5326cea33d8427b45c5f1c5b7bff2bb455767 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9065f0855ba147b109db2e21f9934cc6f57c4a56 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40aa11c5d1f38e6aee1268951603d5fcbd705e3f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38d1e9c256f0aa791db1e945d697dd740d7f0f99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7326514a452183cfcc603c31a8dcda1530229fcc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e865b6bc620b2fbbb79df705cde11f1a982a18c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9457d1bb90ffc2d818a9ca10a762866d2c243c55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e04fb1ff63dfc2d5980103c175d9709e60fd09ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03c9cbac0617afc136a3ab0a1e11d7d64974f3e2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db104be57b25f8efd7f4238ade9ad6f7d757a24a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..729fed2bef95ce20bc22510908d99174ec2865eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13f5621b2fb49aa66933cb9c2a38f8e1f2010065 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1914e1fb86082927733adab77490ffd613c1f93f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa878b037d99406d08fc104b5099366c39eb6ed6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10a136746e32025589512aad8a7e9cdf208ffabe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eebac40961da29be7dd22d2da3895e7efb7cbf3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a22ab84293fdcdf9991cbca6ec33c70094615d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc681f90927f465b5e1d7e95d3c7e889d4f01248 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81bb343c37307ce9aaacaf4f55ae3b832a0c1753 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bc4cc4d9d2c487ced03d857086daf7a768a8052 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5d46e5ec0358ce24e3cc92b571d6608a5d8fb29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4848cc314c8038896b463119b1b02e8bfd42378d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a22c79abaa7ba6e9cd575e6898d09debcf4ad5b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13882279817265b164fab2e49556770333e673d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ff436ba4e0fd6d30c9cccccc1d7efbc19f6f1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a36fb8650e36b4a71a5c9d65cf676cc02be0fab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1c9cc1c43e37f85d3fa9e9de402d925e4b9754d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bc1314ba04e21d323124278e20ae87c4c7b5170 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..259534b9ad217196da25ef9397e136bbcfbfce0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c94a073b9ff6a08893f2bf41b38a690ef95cbb8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ha_stem_tasks +task: global_mmlu_full_ha_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..666d47202e2169eb8aea1b15f7407b1851b2c3da --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9528a1f371cb4df01d394dcbfce4bc1098a2e56c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92f0a408d8079a133586d8936b1fd0b0dd35b477 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc97a8dc9459ce344ff61bc5273c2fea5d152ccf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbcf96c22ddebe8754e274b9d94f3c94aceaea7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa7b426699d3e43acc47f8e2774b0543aff4b992 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b413e4be1fc0e569c7e7289f1dca8c78ef9b7972 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..118e480143d13d4ac69958c2522c6e4364dbb8c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a310d02303ceb14d4cb741ee88617082e4bffb1f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79536ddc615fbf3d6ed30d28f5b810c8b2a73dad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..613170da1f85dff3253bef9bf217b6f422d80e86 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd65c2330ce63aa9a9a126f41ae77225f3a69171 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf7ecb1f5aaa65e80349bd820fedf5e408348242 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9cba53f902a106782a6d8435690ff10ad8cf2fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe767686069acbc9e8f0c51bede57ed047a3329d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94f8e311fd7b3da1b82f3e092c9779fef26871be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54f82b3f3cdf00a80867896657a5659f82c4c862 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ha_social_sciences_tasks +task: global_mmlu_full_ha_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7d224d5c75a43246415ddf9699867c655413ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ha_other_tasks +task: global_mmlu_full_ha_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67a6d33db7f0124e6261a5f31060982f0a0e9f6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/global_mmlu_full_ha_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ha_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ha_humanities_tasks +task: global_mmlu_full_ha_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ha/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4952391a266466761ee29a8f7db864fbfabc4e52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_he +task: + - global_mmlu_full_he_stem + - global_mmlu_full_he_other + - global_mmlu_full_he_social_sciences + - global_mmlu_full_he_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..678ee0d4c308b9e1c3cb492e088f6b7fd881af14 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_humanities +task: + - global_mmlu_full_he_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c99b4806afbfae831bdfc9e0248aea0e5bc7ce71 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_other +task: + - global_mmlu_full_he_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12906895890f10327ca750fc03a6f1a326e59a8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_social_sciences +task: + - global_mmlu_full_he_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6e76e7a87c10d74a947fdb40107148244c582e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_global_mmlu_full_he_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_he_stem +task: + - global_mmlu_full_he_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_he_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_he_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6ec9fc82a8b0747d0042e4654a724b872a1becc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/_he_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: he +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb197c3dee961941080d5b88f1febd1abf6a43b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ab9ee20e2df2ad2769a4ab644f94fb05f788de4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8950b1e039cffddac8bf1182c0ccd3c9be4c517c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c114348b85622a32550d967b55c20f916ddc0eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1324a04dd97d72f8c44b7d659b4ceea1b665e566 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cecddc603f45eb35cfe4ab80cdbd8c27cffa4681 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c0f8b5d1c1e4176d149b3e90653aa1478101ea3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4c36a41443d998c85a309c154316eaeb19c86f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3633d53740d18674d0e8505944628df47fe1b631 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a28c592e502b2a999d8978e3173d6a3631203d2c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3893b9aa04956d4641327bf1bb53e175df65e9d8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4167874ea9a8832ea099fcf860d45acd975599cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ee92851a474e6f93ab2fd4b9f4d6d4ea5b5ba77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ceb32776617c04b74f1cf0897ff85618706ac6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00658e28e44c99519557fc8f94cd6754386aa8f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10f2ac18dd7c0de14e1b86d28cfacce1325be8ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aac3f8dab339e87450c248d5ad7f5f2a7ae4eb98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..299a73efd25f09c7a5f3d711ec5aae57c2fa3fa3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d3ba89356a66c2b305d097e1fb353fe11903338 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f67f8ef3d850e5a2f17a55a8fd4fb4ab9db25c1d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ca8b6f86de781c93eadcf313ccbf8127d8df982 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58e4081a54b25820f74518200f302130aabcbe77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d76e3879d443b2247886eeef09bddc42d5781c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e374511047550cf157d05e52709b408b7a8100db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa0b7c711d20a4527886dd67260fa83b2bc760f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f78a5c59f0ceb158b74ef7d08c22edf3d518a41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15be9243320a614d9c5f04014ceadef1b4d1d50b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f309c0bdf72c16ee2ef039f38fd964b0f6473ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ae831c6048da958d651349f7e9e802c01fc78ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a2e817040d3c3783a88c9b4e2fe734f8560123c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c05da45aac6dba2333c46e2e7fb108d87fc8198d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b818e4fecc02f412354cb8345ecd969af36748a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49f7ce5dad10a20012dab1ffcce2c26243227004 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91d085670e39b875eb15a8f5d9f2a3e4aea1c9c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bedb4f42b654c195491149b1bf8a33c169ee62c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39fe15a2bd819eef516f62dcb4582a2d0044414c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e54b58b33115f87f1f46c8d49c96c7e2753b1cf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8190e96ace0c789304a56e2ea91660eb437aeec4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_he_stem_tasks +task: global_mmlu_full_he_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5811f804315c644c71bd0043d9f7726c78aa404 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fe44232ba6fd70a2d2dde0ada318dfce3d38298 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c9082c5f5482d7a66fadb34c240b333748bc7d0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc419deea25f0070c98939f0fc2e274531b4a8bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d889642bea8454eb5c7aa58fe8a7f7c9f15daa7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1155482347a62cb9e36546c567a3f0a84edd926d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30d49701b51d7dc1942b264143d42b2dba1103c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..458632de43041ebbfe396014a8a72e96a4d3e499 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..938356731ca520c0cd0f80735d510c4cecba4f98 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aed2863685feccdc5acf230100c96ae90492c937 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38a9e3cc2e44e6f0b68205a6251fbe7c0fe8c822 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8ca950c549efaef2a513a48999f8b44228dc4dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f82c289238a8b7f11f0cd70badf857cea4bab501 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3aff6615fe0726df00a3ac8913f815852421886 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e99aa0154cda8d17d602c4b2016e39166df1b4c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de81b92c9f98341d26ebd762f5fbebf51c941265 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7be650440d82469e4a57826af165f71b2706ef73 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_he_social_sciences_tasks +task: global_mmlu_full_he_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6f51e1b93a51544a8cfec7f37f0cde169e6a9ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_he_other_tasks +task: global_mmlu_full_he_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3d10a0d52724e6f969bfd3f0f785592f3806d66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/global_mmlu_full_he_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _he_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_he_humanities_tasks +task: global_mmlu_full_he_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/he/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c899be84ff3458538974232ebd9793694a85af8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_hi +task: + - global_mmlu_full_hi_stem + - global_mmlu_full_hi_other + - global_mmlu_full_hi_social_sciences + - global_mmlu_full_hi_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36492fa3fc1234f1b9996ede7da4f88aa7c76b8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_humanities +task: + - global_mmlu_full_hi_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08dc16b0f679ba0948cf4acdc1e46f23adfed45d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_other +task: + - global_mmlu_full_hi_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a4dfdd726d67dd16dab87e6fcda796eba0709b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_social_sciences +task: + - global_mmlu_full_hi_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a0123ae4a892a3b73962c68774353bb5e54154e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_global_mmlu_full_hi_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_hi_stem +task: + - global_mmlu_full_hi_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_hi_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_hi_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..18c6286eb4489c2763f1d1c82170c20deff664c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/_hi_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: hi +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f239f067d718c93b757df84a058911fad7113d60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfcd776ed8cff6429d2646b068f585dfa8d50254 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbb6763d3e2e00df332526f7452f96bfe28892be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5882427e711ef090cddc5ed6fa9ba10a9493a3b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b0c6c3b0babc254cd924ccdefabbc2e62b589fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5326c8dfb2fd48a423aecef01fe5a42c89e01de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9e21300a6017ceff10fe8dc03b9a2c4aa4314f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c79f4250c810f6e924f51d0bd46355ba1c6d7a52 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e8b0427b43eb4f64661b646aa8795776dfbdfe5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e8c0df2cffdc21b45426faf65d2aaa91e4fe05c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe337ee1fa73fcaddb77fc9979d0a0fa568ed63 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..029a02e07af38d65273bbf4d8fe8edaa53d455e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a674897423d88080d92dd1e7479c81c96396715e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..355053b25b7236db96c18061c5bbb405bbeb79b2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04dca10d70fe4a2691e9322fa1fe77af1bda365f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca7a30832596a2a0e131a02c22ebbaeb43cb1fa6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae534fa62f271be4d8b940342f50aea81f7ce2eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..096fd58bca90fdb6b8683c2e9b96eb62569e19e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ef04ee573cc7d023669de996bbeee1a7e970349 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e8913e5217d8f0ef9244338d8c33ab7deb992d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..180eef7563b937ebb38335a9c48f1750c38025cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32abd63b6e3f1e57495347cf054c739604728d8b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1089908b082a3d6b3d93572850014ac8c16a6c68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb22bb5106a66ec4a5214a00d348c4fa90eaa2a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..affc27c0483782244eec39a806a1933ae2213bf7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59f97c949e8707c0e142309684c907479b108cd9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7506a4cf2fadf837cc6117dcdb557e6f251c839 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..406035bd423464a4af5061f2cfa77318654feb9c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5c2be371487841b9e03f7da21f65dee262ec0eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a955febe331ce3cd339dba9d4ce9240561f4cfb4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a5573f8b315a79b0b5d1a9df64b1b8046adcfb1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38ce46800daee48bcb3eb6f1913c691b9295e5b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2486301f8c7b6ab5123c77bf8fcb2073cd2d4e64 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f889885b32e68a52026110e11f1e440cb04d642 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cb0d834e5d7f54ac1aa316a459cfa2d8e34797b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..113291306246243d6ac0fda1416829fe38656877 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e22cedbe2a477bbf7dfdb3f4eedeb78e3ea22d66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..134ab0806eeffc0d183fe38dd8648edbdf47e5aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_hi_stem_tasks +task: global_mmlu_full_hi_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e523b5d1a0f00cfcc192157c30e42ce849eeacdc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11d8930b739dceda508860ff3aa8164f0cf5355b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad38e3e62fe7ca1b10160a3a579f243529eca7ad --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c31f888329cc7ffc34a597b43b02c009e476352c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01145f6f74bbeb7972d21252acb9ac222c79ad7d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4acbb127b8f67fb74ad04e702538141ee5d2ca5b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..889c0018207be7de113e0dba8b4d26565f91952c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a8aeb4d3cc93d4f8ca29e2b383e65cea853a61c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad80a3c086f2d069a4226834f1cb0ec4700635c3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f5477899b53f51f03731608130cf564c92b886f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836d577d300acb9020fc49e76d746df5c8006763 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a8e7db926b0f687baf4c6314da45d8aa651d7ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4ebc1a5f0f9c86713d68524e448e97d066da95e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bbf959ccbedb6b2d79322af80db22ee05b14cd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7faa9d43d9c82f0a35e6831aed097a3d82d44893 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0ca49ae208ad46ed4d23aec781a3adee1e6eb4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fd9f0ffeaff1811a9d1934170e4c0a867445d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_hi_social_sciences_tasks +task: global_mmlu_full_hi_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..843ea25409dfc301b5420a630d50bfcd9a758e91 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_hi_other_tasks +task: global_mmlu_full_hi_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5e56ce044ac3cebff906ee21e65968c480732c7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/global_mmlu_full_hi_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _hi_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_hi_humanities_tasks +task: global_mmlu_full_hi_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/hi/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd29a2ad18ec549f0feddfb493e8bceecd1740f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_id +task: + - global_mmlu_full_id_stem + - global_mmlu_full_id_other + - global_mmlu_full_id_social_sciences + - global_mmlu_full_id_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9283f55ff308bdb51432c2d300bcc22da015596 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_humanities +task: + - global_mmlu_full_id_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74de0f3607f3c32044a4ad5ecb3d67c67cb29c49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_other +task: + - global_mmlu_full_id_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8656b6bedca7e885e01a03871f183c9749665e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_social_sciences +task: + - global_mmlu_full_id_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0e472764f68eb67064c4e11d99b0388e67dec74 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_global_mmlu_full_id_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_id_stem +task: + - global_mmlu_full_id_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_id_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_id_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..32d9dc92ae31059320ceaf50425421675c6acaa9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/_id_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: id +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b18c1cd7f5ebe0a1e22e4172dce1dba6d1f95d73 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65b83d9d750826041b8f0d024daef58d01e5b804 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11f1047c113fc9d0415c003b35c15c3f246f5b60 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ed992f22d79bbc0e6134cddeda647e9645c1e8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8baa424fa121882c672c5a8d091268e817cf7a50 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67b9c935b942d4cdf0ba35264938695061efa321 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eb5d22804612c2e93a3f580936520c14ce1866d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1462945be52edd5806de1951a35f5285a96c56a5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..980627920d2606e5cb3e2485eafebd7fa7f3dcc9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a2736e891571b829aa205a74b51689df613c8aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb88c3f8cffd13bb80cb56586f94f44577ccff01 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9764ac3efdd13ec63afd28ee8f90e082e6ad9780 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c70c111caf02dd6276ce1af666f98df440e9e6f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f82a74b24a4e9c72b097bb8858c675d01536f37 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cc2dfba0da0b90620b5fbd9624d839dd604fa55 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d511b4baeb397cdbd7840d00c3e461b00b29e0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c6cef13a7346b7aaf7016cbb4118f9d3df9abee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a44daa636b07870c61ce1639db26422b7b446 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d39c31abb9ce693788fff22dc2d8b86269d6bee3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d92d827af1eceb0648c47e2f36bcc03d9eea7826 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff714ac846b2770c34ae7adbda94e92f65874044 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d5c8141851b15325ed0e89740c277c836aa9593 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ad392b3ecc4fcf43906dbfef56ad504e40b34b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850d6d828250476623396242235b809761eb6ca6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1fda5c65ae4b91f0f2c3ded46daeea8d5297cd0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a628ed9a53356ea6adb37bf4e8134458984d8d6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f2c447077f9ebd7e127d0c57d601af3fbe0b9234 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75888a3d00e82373b6eec1e63fbe4628c3aec546 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a6ff54ba6b995cf182a942d7c81b5f0b2ecbee3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab2058026d71b55308832cb07155b60b826859db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dee8c3192c4edb818e01acf2dd4e00f1be27e24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5474c8bacb189560886b46624c8eb0d37154255e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..464ac67f7f857e02ddd204abdcd3fbea8ab667be --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..518cb30c4e107c0da6da587889d999a4a91c6f42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90262ada02b54825d78cea5586f5782e396b4615 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8727ab4916766bbafa4de49e1c900faadcdda4d9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da2c8e6c235015ec9f2a044cc836df650c5cc32a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84a30d9d8be85a0c202fbd31ad4573be6b4a759f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_id_stem_tasks +task: global_mmlu_full_id_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdd340bd88736e9b57544c865286c8b4cf13e29e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caf3eb0f29f2bfe53f523d7755393e2a2c9d589d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d649fd3e41202ec6bf77bcacd7d388e4a2de55e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0811f1b06f88bd4b8151821fe6a918a4206c52d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a124ded50f54ef01d737e47b82b54f1cd633ece --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65dfaea7266ee9c31deb2b19413e283b6457ae9a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..804ffc602c974548470d4f80f08a39bc63d001e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88b37de87a71511f81dad12955392b9249d4dfa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e851c4929704957fd4d784f4e0902573f8994bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d45c9517413020f62e8e4dd8f4a751b40afdb2ec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..965cbad60dfcbb7b8d9fe5cfb856f996512126b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdd02d534ab32e2afa85e3ec2621268e98920905 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d294b27045f1978b60c4149dc95ca9ab12fd33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f772b0cb978831dddfaf8737b28676bd411e82f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a73d36bfb2a02ec092fadef29175569cb2e88d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..715e5c31e635458466de832de462b465d64ab044 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59e147aea00069f3138f1ad8c1a561d641755854 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_id_social_sciences_tasks +task: global_mmlu_full_id_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50225ab5f34367f75ede434f7e5f687045f84eb4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_id_other_tasks +task: global_mmlu_full_id_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0193d12d0bc16e6437b8b105ae6b52af9a42b1fa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/global_mmlu_full_id_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _id_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_id_humanities_tasks +task: global_mmlu_full_id_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/id/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5346e1cb49b7065304d4424659a1afbee0b6b17 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ig +task: + - global_mmlu_full_ig_stem + - global_mmlu_full_ig_other + - global_mmlu_full_ig_social_sciences + - global_mmlu_full_ig_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c6ffb6121eefeba7f838bb111e7f019d6038727 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_humanities +task: + - global_mmlu_full_ig_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..214efed225785ada97a2a0219c8d11fbcbdca191 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_other +task: + - global_mmlu_full_ig_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e27fe1fa179b32b951e30a4f195137a9ee6144d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_social_sciences +task: + - global_mmlu_full_ig_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dd33b62016ebe64174bd8626a9142168e3d7cf3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_global_mmlu_full_ig_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ig_stem +task: + - global_mmlu_full_ig_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_ig_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_ig_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0832c633b740ea5174f877023cbb5002de8eb1d5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/_ig_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ig +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dbf6c8343d742e5bcb8c2db7ba386d82472864d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dc198c25fa1a3241569b6af84f55172ad16d4aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..078069eb22ea1ffc5ddddb63e8a129511df20c09 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f075e7404f9586fe4db598844e631784f7a4f4ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d41779ade0a3af33435afd77c5832eea080a445e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f0e57052c224bf00637d7a40474506d6e31ce29 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78e25dc8d08b54118d5ea0a7adb60e04470f3a36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9894a454777e0761c57c2d35ee3468a5cccb1a9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8976041fd6471fa8f37675e68411c959457f09a6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5edaf0d5609815ef6d84b422b4ff80ac2f6bc566 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e55c01cbd753200f3e07f3ccedf261160beaa61c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ee7564cbe4b5b4f99424a5b88df70414873ce0d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..555d4fa83db09031ce9961777b4753524b1e5ccd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..783804b889a15aa27c8f642267d21ef6630068c2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..789f95d20a29de34b60228d87934613d7d2fa134 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a5c9d2c5551bd384894e4d5d889d11d21bc04c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f9e426ca37bb8f3b0687dc58a5b2763bf24dd1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9b7955c66251e04e5db55bae6927474c3dc065e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..368bc71ddc19dd3702a0fa4bdd29cc0cef712769 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ce77e108b330987057aff3d2cc898aa5fb9b9c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d859f390f2fd59a9e09c6b608a6aa170ab8cbbef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29a93f460b4fc085a1919ba23be8d64fed83e726 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74194a44a8f79f4ad414ad2840767d2248cefa2e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd53504d09d28babcb54e238ffaa4b1e82664ae9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30244a64c3355feeecb73a0077d2a857d7b1a75d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..737c0a56c71f6a19b55d72141a2f8f3d815dbc49 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a2220ccbec2817da79f2f42066c1af4f9a704f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7d4c537f50055076f263979c4a3fa5e23852f1c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3051f013f1ca6d6aee0714e5722fc055a1b1697 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d48410323b57abe316a6ac7beb9f5e331d93c932 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61e124fcfc8810f3023f59bb180997052ced4ee6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d83a63d94732743d85ff87aee3f31aaa96d52eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..787e3151fdb9a57b3c4e92e32a04930c2319072d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c61845911f8ad941f86774f687b17a2abaa6c9e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a8511d215406b568862c08e3e9f5d05effb6bb5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46254ea14c3636103dd882d2f61d666809725b39 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bce7502915d3774927d494fabf2338f5e7c22eb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93c87fbe662456de254a891f330e5b83490d2008 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ig_stem_tasks +task: global_mmlu_full_ig_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..780e1c894ac9987e841e9976cc1245e481636878 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d30ece9a83aa81275a15b79766f25dc8979925e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cac197c78e0d813636a24dcfa3adf2d36f360292 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a38245103d61b773987ccdb1ffc2235704adb796 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc545d84ceff698ad94e4da7ebcf114f0d3b8a44 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60ad22feb3a862a7fbda02ed3e8b3e916c27e8ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cc556070e3aa52a43953dba17d58e719f5cdeb8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f6556324598955b38ba5bd6d516a2aa5975daa7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db4affcff562cc7f26b54ebc6504966ec7958898 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18d3577349f279b26f95790282d1eac115e1aee7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9db41d007aa732ddace06d4a43befb8c93fe8a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fa28b603d3e7617b207cccd464c83007b9b4a02 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..639be38138523058e2ca9756ecc97209eae07eeb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d31af09f4db98c7826584f80a25494ea75806eec --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..200db46b00cd5e83afa3a136c08a03438f36309e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65a3e4e1a85bb47e638b3ae42358f37ba9e9e17a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff0b0505f52ec9435bd5dcd5f9e63ac6ab0317bc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ig_social_sciences_tasks +task: global_mmlu_full_ig_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b437c82f31a01686ddda4e25b14f28e7bb24a8fc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ig_other_tasks +task: global_mmlu_full_ig_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fbc7cfd8237240d2596901daf74f316f0ba5cc6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/global_mmlu_full_ig_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ig_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ig_humanities_tasks +task: global_mmlu_full_ig_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ig/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3bcd69418c599562fb87eabcd6635ed286fc02b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_it +task: + - global_mmlu_full_it_stem + - global_mmlu_full_it_other + - global_mmlu_full_it_social_sciences + - global_mmlu_full_it_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d072ccc57e1c027d5b6038eb94715f01c916931 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_humanities +task: + - global_mmlu_full_it_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99fe18cda989217594ddd42883c3c0b2870f29fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_other +task: + - global_mmlu_full_it_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15a457a5592ab4c8a38c67c828e98c67896a3a48 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_social_sciences +task: + - global_mmlu_full_it_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf7a555dda5f3a6a88e50703448b4063fdeeb99f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_global_mmlu_full_it_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_it_stem +task: + - global_mmlu_full_it_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_it_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_it_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4798e10a9a0eb7132c276c522aa62fd2f256a2c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/_it_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: it +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7351c1eea56ad7d91eef0bc87141d7da1cd0004 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..436cd3f661051384d834f12675aa399aa069052e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f98f0f20edd395b148f5d5f39bb60bb9f7f1d796 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9d931feddf665778f78091cfd672f7c0a4c013a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe4290240ec5ffec9c0d0f95804efa55c595b00e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71b8f45e2b8368c0be2fe2a6d8b26f5bc30ff95a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d29bd758518c8ddc7a412f4abbcc613ba5e0fe8d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f740d259df43bea0ce866be2340e88fa96151b75 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7568fb7e67f44f4755c81dff9cc008807a956ee7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bfc5ac1ba55fed2628a9702b544a1288277b444 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2101847ec748fff0cd4c5dd694a12b38a9a3b468 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70b31f9a81bd66aba60f539c4891e570c8816237 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8917d409237b260f8e8428e56cff1629db8e041 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a49352fb72ab2b57424961b5348f29cfe2e3889c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27f0c6c330cba260e88a2d9bc8a4aade6c13357c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd78a52e84b734b46e754eab94dae7a5f32a7ccc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8171fcf18276aa2fa1c0bdfcabc19b639654ee89 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a952ed444b63398acc137d3ecce77de89fbb21d4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..939ba7525bb022b60211221597c1ca3eeadf0223 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4524d4dc3259c52e9f7cd0e339489529781c315e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dfb1649fc16f0eeec93994333264e84f2a3f3fe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..556aaf20a319458b6e4d3a4bdd13868af84b61dd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c1d5b6008944c8fc498a1b873b8f7ff87e8579c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a35b6bac89dffb1337f7782e91a0119d288178e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74c01ccd4481afc27dbc485cb5b09b7cfff43077 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bec02c4ca085d2e6529c0fa0ab7ae57fc1d06f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..551a0f8da6f8e35b523756572f73bf7a2e0bb27e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cf7144bdda5e118f79bd78b683eab29df2c8325 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17088e513c2d51ec3ee51f3cd4713a76bdb05440 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3f35f991c13b7243175f44ddc87f7c1c3c4dcbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..698ddb5fc3913b21ad05a94c5e875179b44e0698 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ff49730e275d7a68041039ace5f155b804ea8f2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7c47e55d4f495dea81647b1b4a460282706de77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e100c0e9255e85c15f3fc87267f1228a74f06ddd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a07444a85b407c3acac51913c97eaf604bfa5d7b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfd3b7a5ab60d33fd4dad138a5ea501f3538ff15 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_it_stem_tasks +task: global_mmlu_full_it_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b5feeacfc53a4bf7df581d5a7907344ba52450e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d50b46f44e9fa6b7d5287143e01b05c07b800ea7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b02316caa27431e6cb90aec3cfea45efc89c85d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b638b50de1cbcc90c682eb0017fd89f74df2a040 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520a8beaf092facdd2bb080addebf312d9d952aa --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abfc739509650b46ab19238d23e131f8bef6daf9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cac74152758293800245078fe7168d680c213d41 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1d94976dcda73baf2bab8d6e910c5d97062d415 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74bdec821927ce39c943da94f5a7f4c5d6e1bb06 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acf999a2947ad8231d79399cc24422f7e6b8b851 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ec4f58e3675b456e3ebedaa36c49ce45f9a4560 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_it_humanities_tasks +task: global_mmlu_full_it_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b53cdacb8fecbd3860503d04ea8fdb78f86c860 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_it_other_tasks +task: global_mmlu_full_it_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90fd186c56ea787eb26099c0e58950d79472ed28 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/it/global_mmlu_full_it_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _it_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_it_social_sciences_tasks +task: global_mmlu_full_it_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af25573dfe8cfe351e2e7cda538cf4129b9ca1ef --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ja +task: + - global_mmlu_full_ja_stem + - global_mmlu_full_ja_other + - global_mmlu_full_ja_social_sciences + - global_mmlu_full_ja_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a063eb0b663cacbba1b79afc5502efe5e5681ee6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_humanities +task: + - global_mmlu_full_ja_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4207fea43fd0cdace8edf8182bcac5b57d1e6b24 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_social_sciences +task: + - global_mmlu_full_ja_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ca6ed1c06a4fd65c075816082fc9bf2abd5b0f7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_global_mmlu_full_ja_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ja_stem +task: + - global_mmlu_full_ja_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_ja_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_ja_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..591725e38e184cb3d173071b0d9aeea97948bcc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/_ja_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ja +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b65a75be2476d4dc8bc1244d7b26f8c04f8ad99 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e735aa349df3f7c8bfd787044b2061add9057093 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0ba89475b74e0f3d4492db08baefc458bd9656c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c39d286c7ffba693f61da3749be7bc43b03d858e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a91a7d61717a3c2ca053a826efc3026234aab59f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67dcd2a02289efdd3cbd150bacb2f43d2cd83d25 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c55ab2a25dee6fa229f5488a423cf276ed981291 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ja_other_tasks +task: global_mmlu_full_ja_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f823ac44b333e51e4570ea77784cd02818d0a4ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba7295759a40b51bf0d066b8081258f7a87a24d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ja/global_mmlu_full_ja_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ja_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ja_stem_tasks +task: global_mmlu_full_ja_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e617e8cdd6144066c57ece957ca27f0e1493ffba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69e3a2df40e209b8105e28c08c1eb63656fb4917 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..651f389c20d99fd93cab22351d71ca13ab890f87 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..001807ebcabd19691bfc34597eb9d0528a7a549f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1126c6b7ec6683b245df8f76d16bef755640eed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ko_stem_tasks +task: global_mmlu_full_ko_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b8332705879711a026ea9df177fdb51f8bad090 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65df17861b5182a5c75e90fde31646dd068a37ca --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f1e7fa1bffacd485db4c9d5c556e331c63c7abe --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c657543aa52eb0e75a7d2a64fa959a02c3a3a478 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff6450f921d469568cc3f1d3e7655e090b2bd0f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21f058af0efa215550dd81539e55c44fbe63fb6a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56aedae91fa5a7bc214dbb58c2ad3eae52694b33 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f83b232ec63d61785c9a4487d597295702800b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ece9dc5c6ee09e19f6d02523b1a38feaa62f06c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..439309570bb0360304ddc46e39356cdb8ebfab7c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98ff6520d1b3037f8184afd6da45b1b8ac2ab233 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a5b07f7e31cdac2c3417e9ae1c6de2bdf46bd42 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3663391a8fff81cf98712224e039b83b389caa85 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..902b4443e2c696a992c8d4089604e7da23adeb3b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36e1794ceb2bbb5737717d7cb8bbc2084d1e861f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ko_social_sciences_tasks +task: global_mmlu_full_ko_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64b58d6a95907577fd3f072cb0348bca59c80656 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ko_other_tasks +task: global_mmlu_full_ko_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7289671f5c4f6942fa770a835fddd60f3ab4fade --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ko_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ko_humanities_tasks +task: global_mmlu_full_ko_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ko/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33b2a4df9ebc7f8897d623ac2dcd20670fc131f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_ky +task: + - global_mmlu_full_ky_stem + - global_mmlu_full_ky_other + - global_mmlu_full_ky_social_sciences + - global_mmlu_full_ky_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e0368c2b6c777c29b37518cf7f574ae06a703d3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_humanities +task: + - global_mmlu_full_ky_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1bfc89ab6777fbf1380bc4ec506ac2e6ccaadd32 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_other +task: + - global_mmlu_full_ky_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae756c4bc364d0aef2a8b093b3b27f1d603b817 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_social_sciences +task: + - global_mmlu_full_ky_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..817456fcb2c31df7614c2dc8973b9aac83ed92c5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_global_mmlu_full_ky_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ky_stem +task: + - global_mmlu_full_ky_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_ky_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_ky_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..63f88823861ee5dcb129d843dc1ee45095ed93e4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/_ky_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: ky +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21338a569ce25b419e959f73732269d734a19d8e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df2635483c6c407c8756bd25151143f87fece8c6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e0f6abab0550e86425933ae919f5f185359c2ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17656dc6362b31b43686f1d0c6640219365a0854 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c053b884c0029da3bb76dc1ca3f00c9a0df0383 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3649210635296607da1a355402b61731c1d064de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9f858681b003e4cc7670866ee8c99dedb63e3c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4b15b5401fb69a53a16b73369019578d4c656b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5657b66bce4b9108fff6a0e851c827ea1e3023a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1e0c25e6c4f9685fb37b4aa4446615714f15f04 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fac1d80f31fe382a056a9a2be7d60208c7d413bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e35718d028c1bf3aa1a1be071e5441ff42293b0f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f165ec61962550c284827e68cc905d22f31568ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48670c7f8d17747e03a11603c61f563ca717fba9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29d241428f1984fabb2071cd248ead66bcd99ff7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9aa7f81b48588e4c43d03108a5af42ab13c431ac --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70a5bd8603a71d3f9d57ca9265f68d1410bc70cc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f678c0d130bd4de90abb3016fbdbc18e7384eab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..750bc68b8c8c769f35832e610a1f8c11d9cf15a4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7700e37fa9d1118af9916cfabf9564c44e07894d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c805fc4e91c7d29dbb2bd33ecfea8c8c116639cf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01c67f8e332cd612a810f4801e3cb16211e7eb97 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccc5c8b73d1d1e70ad5c12504acf77fcd15c0273 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02ea66ef6b4be7105afb290010e535be8244e7d2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f693296dbef784f629fabb544cc95ba3f7dd4630 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b05e279925c8fc21efd67700618d830a5eea9ddf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d596290fa3ad19f7641a0dd178291f00e21d341c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f71865c250b4424f3b4800b19ee176adcb06d4b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..635873a1f1598a3714c7b5009a05750ef3e26039 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df8cfefbcb7d17314d05930ee8fcd140b2235025 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c75f534138a80453eee56e52a5da632cc06835d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e73edcbea8ae601b1fdd24b8b0dea1d9a4222d1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4e662a56d1752251a61149728c59059c6591925 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c2556da1976fd1cd586ecc1a1d72be01fc84caf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2af16190992c5f22249fd716fb3a029e0fba1aa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0994cc2d715489ce4cca164cc2ae19902e251ab --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7933a77b34683a2f9d3d2557f1e13fd23f1c072 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6e525a18a939d63290ae9830a78b8f840ae5388 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ky_stem_tasks +task: global_mmlu_full_ky_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03f70aa02dd866cffc52849bbe0771933557f10c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72ced798867ea4f86e9d7bf0717608be99da70c9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..371e4b2176d9396677782c888a954a1acae2c1e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e693ab8b9f1a4c5e8d149704a0bdb102c7692916 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccafcb1e0e6f06576f38b0b4e38cba94fbc1abe7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16c19b29eb879b3fedb7f9f8411d9e9f918bb109 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6c00cb0e7cc9a906dd01a4c25e663d9cf09908e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d6d242b0181a2b7039f5998d6a02108a90bf585 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ff2e08d4ae0c4291b663320d9ccfc918bf6f8ce --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37c6a892286f4ce10cb682e60829b525b33fe54d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b4fea0e5e3b23b9b3623e7be6fa79b98db69592 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ky_humanities_tasks +task: global_mmlu_full_ky_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a6ef0b531d1a185be293f35b82c18340e39577f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ky_other_tasks +task: global_mmlu_full_ky_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dce1b6d2d626b35d3c497e9a6e8289017348c39a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e24b8167a177e2a5cbae1dee58e34ec2eac20a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ky/global_mmlu_full_ky_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ky_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ky_social_sciences_tasks +task: global_mmlu_full_ky_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd2cd37d3ee9e4c7a1efc0862068e4444f6531f9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_lt +task: + - global_mmlu_full_lt_stem + - global_mmlu_full_lt_other + - global_mmlu_full_lt_social_sciences + - global_mmlu_full_lt_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48ad351f725275c8bafa0cf32b44cd3ccb2cf134 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_humanities +task: + - global_mmlu_full_lt_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f63c35ad2223dce8e1edfd1b4e6f4fd2539c9db --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_other +task: + - global_mmlu_full_lt_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ababd6d14e6b2745ceac16953b93297bebc06c8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_social_sciences +task: + - global_mmlu_full_lt_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a59e683357b1994872a4ee00e498cc413233496 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_global_mmlu_full_lt_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_lt_stem +task: + - global_mmlu_full_lt_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_lt_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_lt_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b925338059be1d355a867e55da838e169ea1d13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/_lt_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: lt +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76b96844f696cc22614cf2643858d4d6c9417754 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..419b89e30ed8d5100d83d56a2d0be71888386e65 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c51daa226b39284c5bf96ec531dc90fb1430fbd6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e023277406ff3c783c095e6996ac0c38211068bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6fea6f851074e422ab06a10afee17138ee8c607 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93b9a5610e16b589dd3e7e68f847490838e7f16d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d0dcfdd9ad41771c5840124f3688e328b35b076 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d33b747d8276d3baa23b7187aaa5e2bfc9f7939 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad74dbb15fcfdbc529e9e0440bf2122ee0aa9e8a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_lt_other_tasks +task: global_mmlu_full_lt_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d78f3a545da42447e590ef4f0156e6ae49f505d1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e7b5e49e6e2028dd176336874c902ea68fccb5f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..284dfe9e27a30ed0e529e80d004c3f273bb2b575 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2324bb288496581d56df81e035f2a0faf16e0040 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/lt/global_mmlu_full_lt_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _lt_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_lt_stem_tasks +task: global_mmlu_full_lt_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3d5921a8ad8139a267cefa8d3b9850d0cbaceb2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ms_humanities_tasks +task: global_mmlu_full_ms_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c07cbdee8cb4a814d567d47b3cb20407ce95ba78 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ms/global_mmlu_full_ms_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ms_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ms_social_sciences_tasks +task: global_mmlu_full_ms_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbc3bacd0839d6c0fbb8b86eb7501c1f92b303e8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..225da2fb5bd475a35868879e26670145d6e0b880 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f5e9f1b4e447bb7658d73e7f48e19b16d55ff7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cff5ba64b352c0d7f391492e95d173e641144f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f7eb3ee33e2449dcafee69e07263f395fdd72b0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0453e51d9d668942973887dc64ea15d0d93584a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05710100729043b043e6b6c4c8e87e418b168e36 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd68d5f041e914b82b4cfc5df2b2ce0f21baa2a8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39ef0a58f7fd865822aa293ec6becd3e74559809 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..535a391821a13f937d59174163b0a481758969ae --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f355dad253203b92e2f7903d12c8353178ab3232 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a52d7a017bf063a16cde6b3a65866785a65c4dbf --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a2564200d4cdc0a0b0c5dd68ab4421861f1c06c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e1199b17fb77ff7bf7c7eef871c8fa2d43fe5ea --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afc2135b37ddae395d6d4f0864f2ce42e23a535b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18450534654c637b06c56836a9eaaa44662718ba --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d23b8397bd04a04b2a3d50f750b39996cfdbbcb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5be599d20c20a2b87ee2043abe7df0d07eca7326 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..180a397c62f976822662b1b3997b352b8bf09d23 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3aa369a96354c77c7e6ee0fb2b9e57f99887d8b4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e08abdac804b60935363639309106374cdaaef4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_ne_stem_tasks +task: global_mmlu_full_ne_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e44c5be63b6bec1011631761b40c32f93c4b775e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10f7daa274c1b821b33b73878f22a186c5629715 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8139b1f7a65c5ff5668d79e8d3f0b19cf1d866e9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb1bf905061a29950ba1ed2b50678fc8b25cc489 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b74fb365022fb4eda448dc1fc77258780c49428 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91f8f06c5a769c701fdf7f7bbfd0955d9b837a05 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..575f0e45506432d31b4c34c5d602b449a1e3d8f0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_nutrition.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_nutrition +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_nutrition diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95fdd0eb343fc2a738c900782c9ce37dcb8c429a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6e5c706c0b37d1d0a176f0d43e6fc41c8a47485 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_prehistory.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_prehistory +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_prehistory diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..718cedee02328945a0e05bcb590ab2ba158af9f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89c70160a6bbbba493f5e3c702ea02ebd850f94e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a366e0c43a3ccadd06b6e645d3f11da62cb67486 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..649e53435a1e0842ed0280efd91ed9ed8c32c822 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37f2ddeab876d0c1861475dafb53c52cb20e673c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55f809041b2b726de52d63ebecec0a584f361317 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_security_studies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_security_studies +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_security_studies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78161d5a113e867abf882a1824a6a6a8fa590d4b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c38f59c405afe4b39415a5ff16d6f4083a23be1a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_ne_social_sciences_tasks +task: global_mmlu_full_ne_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c15808fdc84a2c3b216833352c94731e83cb64d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_ne_other_tasks +task: global_mmlu_full_ne_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c6163f12c9d3d90ffb4b9576eac1f809bb37e03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/global_mmlu_full_ne_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ne_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_ne_humanities_tasks +task: global_mmlu_full_ne_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ne/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c293a558a102282b53df954335faf7bb474f69b7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_nl +task: + - global_mmlu_full_nl_stem + - global_mmlu_full_nl_other + - global_mmlu_full_nl_social_sciences + - global_mmlu_full_nl_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..656a421bfb7dc29eebfe0013b7c69b331fe04c6d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_nl_humanities +task: + - global_mmlu_full_nl_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23a42201ec4f8212e836727b434804977111cf7a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_nl_other +task: + - global_mmlu_full_nl_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afba5678d182bfcbcef3bf227a1800dfda12b535 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_nl_social_sciences +task: + - global_mmlu_full_nl_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9658b13ed512182d510fd72c60cf1eb44eec5e6f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_global_mmlu_full_nl_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_nl_stem +task: + - global_mmlu_full_nl_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_nl_template_yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_nl_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..39efbfd177533bc57bdb3c831385a5c386b084e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/_nl_template_yaml @@ -0,0 +1,16 @@ +dataset_path: CohereForAI/Global-MMLU +dataset_name: nl +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..458a36140a17fca9a855c4c1c2852fba05f14b7f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_abstract_algebra.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_abstract_algebra +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_abstract_algebra diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4cbd90ef070f7bc12149fb03f46c55d6cbbf591 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84cdf5781ba1c8e50b79069ca291270ab5f742f4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f75776f24d231111aa39e224cfeb5e9700b3e253 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_business_ethics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_business_ethics +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_business_ethics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e963d0f0d93ae41918a86b2fb5cf63ef243f5cd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4a3660b07b1681b9f31300e904391ddf57b04fd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_college_biology +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_college_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa9faed92b3f8c53a714cad2294eb6bd737ca57f --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b603c309105a1873b5697c7852a15afcafa3d982 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f55207ea49e6588dbfdc36534ed59f514a3d35e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_college_mathematics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_college_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cdda1b7755bcfc2e080b9c5bdd924bdb561c189 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_college_medicine +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_college_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26d70230fb8b945e0554ff7df68e0b4e5fb4e114 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_college_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_college_physics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_college_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01a8a74742e44468220cf1a7f85aa68f1d53fa13 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_computer_security.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_computer_security +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_computer_security diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cccd266651a36b14d85ece24d3d639964bd7106d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_conceptual_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22ad59bd462ceee61f12955852634723b02852a1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_econometrics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_econometrics +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_econometrics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3aca226fdff9fe7219463f5dcc2cc41bb4fa16c1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_electrical_engineering.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_electrical_engineering +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_electrical_engineering diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2118a1d5d6935b9ba7603b393babba2a027536c4 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_elementary_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_elementary_mathematics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_elementary_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fd86105df589b16c08f9c8abfdfc7eca6cbf712 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_formal_logic.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_formal_logic +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_formal_logic diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7147d517cfcfd02013b3afbf8ba0ec11e95d180 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_global_facts.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_global_facts +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_global_facts diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..271b54f65e95485e1fdfb3b4e3d39d282f6c5f1b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_biology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_biology +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_high_school_biology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..921abd1745e94ee09467c2d63f0575455fe1d750 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_chemistry +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_high_school_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea190bea8face93cccc33b337750bb39707ac8f5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_computer_science +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_high_school_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c348d482a074bcc6dba6eddef8905a8df859cd8c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_european_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_european_history +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_high_school_european_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de31a63b8df61eee7f08b5a96e283c7e5b93d400 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_geography.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_geography +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_high_school_geography diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc0e3cb1bc79dc6b5400497c07ba789b72cec23e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e221c685fb2d15b7d9719e2e17d25ec44b70c1e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..137158a690d175135c0ebf2d94f038ef54fbf615 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27b426c06f46f522fe70dccf7241f05b27af505a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..746df49e81b176db860a12d31781a06ae6e93448 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89cb42d9472a8e6517a6eb5de45312ecb826e02a --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e27082c4cb8c320b8dab7e61d92db36c46cf1484 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_nl_stem_tasks +task: global_mmlu_full_nl_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66efc58c9673f3afb5129caa39cd06fe31e275e0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83b653454773bf44e5a8fe8ad35125fa68b120de --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82e00b4b8fa93e31a3307d40482e1dc628f6a16e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..468589da275b9236dbb72c6eaee686c5c9fdcdc2 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5bf62a98e7b3442f821bc3209d124e061e49052 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81658e9f0278dc681e525ca52f3abd6bc8f5d657 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8e52c0d4ec5d2231ea89155dde624887acbd6f1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31af482ec8ca1a1bbd97cd82f07651b2a35b092d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..853de0c12fb9416a94ce8e8e1bd04e3a63d2ffa8 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b86e04500b404fa145fe1bbaa70ac9e36bdc6d7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e827dd19b3e6184f0868c7f26bd1e38eddc420 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45484116ddcb28ead1fd09b72aa7c169faa5c0b3 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be586b45d96457457c2a3ecc27cc8dbc06ce4f26 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ffe584887a64f84ce5ec1a34c9a83e9c7b8f1bd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_public_relations.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_public_relations +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_public_relations diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6b6227bf6c656c75e4f7d35991761c92e72a19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_us_foreign_policy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_us_foreign_policy +tag: global_mmlu_full_nl_social_sciences_tasks +task: global_mmlu_full_nl_us_foreign_policy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92d1973bb57dc8c7f2a57f8d0d14547b55229041 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_virology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_virology +tag: global_mmlu_full_nl_other_tasks +task: global_mmlu_full_nl_virology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8c2eccaa24955da352770a507f00e59fca5dfe6 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/nl/global_mmlu_full_nl_world_religions.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _nl_template_yaml +process_docs: !function utils.process_world_religions +tag: global_mmlu_full_nl_humanities_tasks +task: global_mmlu_full_nl_world_religions diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51b90446d1fd2362581c3ab9fb159e3fe44d8ab7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ny_other +task: + - global_mmlu_full_ny_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b711dfdf14029a040e965c36fdae79e0702c31e7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_social_sciences.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ny_social_sciences +task: + - global_mmlu_full_ny_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99bf9d95df56a763b2bca25c05d20b1b4f2ba366 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/_global_mmlu_full_ny_stem.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_ny_stem +task: + - global_mmlu_full_ny_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60806afc61e6e35998b3c96d2631b01b66121923 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/ny/global_mmlu_full_ny_anatomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _ny_template_yaml +process_docs: !function utils.process_anatomy +tag: global_mmlu_full_ny_stem_tasks +task: global_mmlu_full_ny_anatomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05a7de9bfe7e12ea3a0eccb781e53b256de93d4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_government_and_politics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_government_and_politics +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_high_school_government_and_politics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aceda633f9b60feae440661fcbd31ac92f5bd490 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_macroeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_macroeconomics +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_high_school_macroeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6eef2cd8578fdcf1ac0d8e12584b576adafab1ee --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_mathematics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_mathematics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_high_school_mathematics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5adb5fa19bc7b88102d43f3b034b39bd60f794df --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_microeconomics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_microeconomics +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_high_school_microeconomics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbda792097d4738c0b6890d538dc56ae5944e6e1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_physics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_high_school_physics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7eb09362a7c24dc436ca10eca7ddf603d8ae70ed --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_psychology +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_high_school_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7beef5b06ee6122943e89c7585778011d3a7089 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_statistics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_statistics +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_high_school_statistics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08f45dd9c748303c2c76ba46da529ca40ab562b1 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_us_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_us_history +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_high_school_us_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99664de8772599a1962da0318d6afef765876b0c --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_high_school_world_history.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_high_school_world_history +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_high_school_world_history diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d63f6f8d4af5ab8920d87c2faff92879cb639dbd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_aging.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_human_aging +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_human_aging diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8080ca8d0ebcb5d7f503d16cffaa98633c763c03 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_human_sexuality.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_human_sexuality +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_human_sexuality diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..425695c1d5c64cd8b350c944e87a40027779a6dc --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_international_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_international_law +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_international_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6455bd71724395745a31dac3e9c5fbae81b4469 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_jurisprudence.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_jurisprudence +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_jurisprudence diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1359b3a24e3ca52394fb0d524c87b3b7a80b067 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_logical_fallacies.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_logical_fallacies +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_logical_fallacies diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d7bb0dcc725fbb2e830772e759c44b0bc1f21a0 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_machine_learning.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_machine_learning +tag: global_mmlu_full_pl_stem_tasks +task: global_mmlu_full_pl_machine_learning diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f695226c1bdcc2a798f8595a06c12c529cfa269b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_management.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_management +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_management diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fedcd3f8817be8bf7be01c9d3157cb18200cb77 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_marketing.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_marketing +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_marketing diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89da9f67565089e10cb852df563aa289afa0d455 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_medical_genetics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_medical_genetics +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_medical_genetics diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f34762c99cb9bbc6d273c016f4e74c91baa938b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_miscellaneous.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_miscellaneous +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_miscellaneous diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25f201f4b6f16809a13a627027b1a0acfac21d46 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_disputes.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_moral_disputes +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_moral_disputes diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd08e6e13257a3a14a50e49c52212baca4c1f19d --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_moral_scenarios.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_moral_scenarios +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_moral_scenarios diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c1bf6dc13d80759fd4aedbc131bd5a1f7777989 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_philosophy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_philosophy +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_philosophy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..514b04cd1167e1b455d256bd4bd0aee450a53ac9 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_accounting.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_professional_accounting +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_professional_accounting diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99c719f83c36178ac6aa873284d2c24742d6cc66 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_law.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_professional_law +tag: global_mmlu_full_pl_humanities_tasks +task: global_mmlu_full_pl_professional_law diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dfafb254e809dc16186176bb61d07cccce71b45 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_medicine.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_professional_medicine +tag: global_mmlu_full_pl_other_tasks +task: global_mmlu_full_pl_professional_medicine diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b6181c0c91f62beeba5d6f9ef2aa561438a1f0b --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_professional_psychology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_professional_psychology +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_professional_psychology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bc0fd8ff08ed950cfb842f368a0a9d8c7707b19 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/global_mmlu_full_pl_sociology.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pl_template_yaml +process_docs: !function utils.process_sociology +tag: global_mmlu_full_pl_social_sciences_tasks +task: global_mmlu_full_pl_sociology diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/utils.py b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7df72cb061f0fecba46e15ff9b57552817979afb --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pl/utils.py @@ -0,0 +1,73 @@ +from functools import partial + + +SUBJECTS = [ + "abstract_algebra", + "anatomy", + "astronomy", + "business_ethics", + "clinical_knowledge", + "college_biology", + "college_chemistry", + "college_computer_science", + "college_mathematics", + "college_medicine", + "college_physics", + "computer_security", + "conceptual_physics", + "econometrics", + "electrical_engineering", + "elementary_mathematics", + "formal_logic", + "global_facts", + "high_school_biology", + "high_school_chemistry", + "high_school_computer_science", + "high_school_european_history", + "high_school_geography", + "high_school_government_and_politics", + "high_school_macroeconomics", + "high_school_mathematics", + "high_school_microeconomics", + "high_school_physics", + "high_school_psychology", + "high_school_statistics", + "high_school_us_history", + "high_school_world_history", + "human_aging", + "human_sexuality", + "international_law", + "jurisprudence", + "logical_fallacies", + "machine_learning", + "management", + "marketing", + "medical_genetics", + "miscellaneous", + "moral_disputes", + "moral_scenarios", + "nutrition", + "philosophy", + "prehistory", + "professional_accounting", + "professional_law", + "professional_medicine", + "professional_psychology", + "public_relations", + "security_studies", + "sociology", + "us_foreign_policy", + "virology", + "world_religions", +] + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["subject"] == subject) + + +process_functions = { + f"process_{subject}": partial(process_docs, subject=subject) for subject in SUBJECTS +} + +globals().update(process_functions) diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cb9f0fe3778e4870b33733795e0e2212a282f68 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt.yaml @@ -0,0 +1,11 @@ +group: global_mmlu_full_pt +task: + - global_mmlu_full_pt_stem + - global_mmlu_full_pt_other + - global_mmlu_full_pt_social_sciences + - global_mmlu_full_pt_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..261a70288e94b635e3f41e5a4b6b2ca58d971368 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_humanities.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pt_humanities +task: + - global_mmlu_full_pt_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a61b12f504501c34dcfa4a67eff3dedc5e88ec4e --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/_global_mmlu_full_pt_other.yaml @@ -0,0 +1,8 @@ +group: global_mmlu_full_pt_other +task: + - global_mmlu_full_pt_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90880cd077e253cafe68709633b533a02a8b0b51 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_astronomy.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_astronomy +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_astronomy diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2999a02a756fa463d199050fdbd1cd579f747dfd --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_clinical_knowledge.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_clinical_knowledge +tag: global_mmlu_full_pt_other_tasks +task: global_mmlu_full_pt_clinical_knowledge diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91d8cd2ed1e520094c8b3253cfd17df2c6f1dfc7 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_chemistry.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_college_chemistry +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_college_chemistry diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68592aafe5b5b00c38bad52edde00f8535a32fde --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_college_computer_science.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_college_computer_science +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_college_computer_science diff --git a/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e6e91a9935da462b6f5bc101fecc2216f3a68b5 --- /dev/null +++ b/lm-evaluation-harness/lm_eval/tasks/global_mmlu/full/pt/global_mmlu_full_pt_conceptual_physics.yaml @@ -0,0 +1,5 @@ +# Generated by _generate_configs.py +include: _pt_template_yaml +process_docs: !function utils.process_conceptual_physics +tag: global_mmlu_full_pt_stem_tasks +task: global_mmlu_full_pt_conceptual_physics