Add files using upload-large-folder tool

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/README.md

@@ -0,0 +1,70 @@
+# mmlu_pro_plus
+
+### Paper
+
+Title: `MMLU-Pro+: Evaluating Higher-Order Reasoning and Shortcut Learning in LLMs`
+
+Abstract: `Existing benchmarks for large language models (LLMs) increasingly struggle to differentiate between
+top-performing models, underscoring the need for more challenging evaluation frameworks.
+We introduce MMLU-Pro+, an enhanced benchmark building upon MMLU-Pro to assess shortcut
+learning and higher-order reasoning in LLMs. By incorporating questions with multiple
+correct answers across diverse domains, MMLU-Pro+ tests LLMs' ability to engage in complex
+reasoning and resist simplistic problem-solving strategies. Our results show that
+MMLU-Pro+ maintains MMLU-Pro's difficulty while providing a more rigorous test of
+model discrimination, particularly in multi-correct answer scenarios.
+We introduce novel metrics like shortcut selection ratio and correct pair identification
+ratio, offering deeper insights into model behavior and anchoring bias.
+Evaluations of six state-of-the-art LLMs reveal significant performance gaps,
+highlighting variations in reasoning abilities and bias susceptibility.`
+
+Homepage: https://github.com/asgsaeid/mmlu-pro-plus
+
+### Citation
+
+```bibtex
+@article{taghanaki2024mmlu,
+  title={MMLU-Pro+: Evaluating Higher-Order Reasoning and Shortcut Learning in LLMs},
+  author={Taghanaki, Saeid Asgari and Khani, Aliasgahr and Khasahmadi, Amir},
+  journal={arXiv preprint arXiv:2409.02257},
+  year={2024}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `mmlu_pro_plus`: 'All 14 subjects of the mmlu_pro_plus dataset, evaluated following the methodology in mmlu's original implementation'
+
+#### Tasks
+
+The following tasks evaluate subjects in the mmlu_pro dataset
+- `mmlu_pro_plus_biology`
+- `mmlu_pro_plus_business`
+- `mmlu_pro_plus_chemistry`
+- `mmlu_pro_plus_computer_science`
+- `mmlu_pro_plus_economics`
+- `mmlu_pro_plus_engineering`
+- `mmlu_pro_plus_health`
+- `mmlu_pro_plus_history`
+- `mmlu_pro_plus_law`
+- `mmlu_pro_plus_math`
+- `mmlu_pro_plus_other`
+- `mmlu_pro_plus_philosophy`
+- `mmlu_pro_plus_physics`
+- `mmlu_pro_plus_psychology`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?
+
+### Changelog

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_default_template_yaml

@@ -0,0 +1,33 @@
+dataset_path: saeidasgari/mmlu-pro-plus
+test_split: test
+fewshot_split: validation
+fewshot_config:
+  sampler: first_n
+  doc_to_text: !function utils.fewshot_to_text
+  doc_to_target: ""
+output_type: generate_until
+doc_to_text: !function utils.doc_to_text
+doc_to_target: answer
+filter_list:
+  - name: "custom-extract"
+    filter:
+      - function: "regex"
+        regex_pattern: 'answer is \(?([ABCDEFGHIJKL])\)?'
+        # regex_pattern: r".*[aA]nswer:\s*([A-L])",
+      - function: "take_first"
+generation_kwargs:
+  until:
+    - "</s>"
+    - "Q:"
+    - "<|im_end|>"
+  do_sample: false
+  temperature: 0.0
+num_fewshot: 5
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/_mmlu_pro_plus.yaml

@@ -0,0 +1,23 @@
+group: mmlu_pro_plus
+task:
+  - mmlu_pro_plus_biology
+  - mmlu_pro_plus_business
+  - mmlu_pro_plus_chemistry
+  - mmlu_pro_plus_computer_science
+  - mmlu_pro_plus_economics
+  - mmlu_pro_plus_engineering
+  - mmlu_pro_plus_health
+  - mmlu_pro_plus_history
+  - mmlu_pro_plus_law
+  - mmlu_pro_plus_math
+  - mmlu_pro_plus_other
+  - mmlu_pro_plus_philosophy
+  - mmlu_pro_plus_physics
+  - mmlu_pro_plus_psychology
+aggregate_metric_list:
+  - aggregation: mean
+    metric: exact_match
+    weight_by_size: true
+    filter_list: custom-extract
+metadata:
+  version: 1.0

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_biology.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_biology"
+task_alias: "biology"
+process_docs: !function utils.process_biology

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_business.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_business"
+task_alias: "business"
+process_docs: !function utils.process_business

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_chemistry.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_chemistry"
+task_alias: "chemistry"
+process_docs: !function utils.process_chemistry

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_computer_science.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_computer_science"
+task_alias: "computer_science"
+process_docs: !function utils.process_computer_science

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_economics.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_economics"
+task_alias: "economics"
+process_docs: !function utils.process_economics

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_engineering.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_engineering"
+task_alias: "engineering"
+process_docs: !function utils.process_engineering

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_health.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_health"
+task_alias: "health"
+process_docs: !function utils.process_health

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_history.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_history"
+task_alias: "history"
+process_docs: !function utils.process_history

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_law.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_law"
+task_alias: "law"
+process_docs: !function utils.process_law

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_math.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_math"
+task_alias: "math"
+process_docs: !function utils.process_math

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/mmlu_pro_plus_psychology.yaml

@@ -0,0 +1,5 @@
+description: "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n"
+include: "_default_template_yaml"
+task: "mmlu_pro_plus_psychology"
+task_alias: "psychology"
+process_docs: !function utils.process_psychology

lm-evaluation-harness/lm_eval/tasks/mmlu-pro-plus/utils.py

@@ -0,0 +1,63 @@
+from functools import partial
+
+
+choices = [
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "G",
+    "H",
+    "I",
+    "J",
+    "K",
+    "L",
+    "M",
+    "N",
+    "O",
+    "P",
+]
+
+
+def format_cot_example(example, including_answer=True):
+    prompt = "Question:\n"
+    question = example["question"]
+    options = example["options"]
+    prompt += question + "\n"
+    prompt += "Options:\n"
+    for i, opt in enumerate(options):
+        prompt += "{}. {}\n".format(choices[i], opt)
+    if including_answer:
+        cot_content = example["cot_content"].replace(
+            "A: Let's think step by step.", "Answer: Let's think step by step."
+        )
+        prompt += cot_content + "\n\n"
+    else:
+        prompt += "Answer: Let's think step by step."
+    return prompt
+
+
+doc_to_text = partial(format_cot_example, including_answer=False)
+fewshot_to_text = partial(format_cot_example, including_answer=True)
+
+
+def process_docs(dataset, subject):
+    return dataset.filter(lambda x: x["category"] == subject)
+
+
+process_biology = partial(process_docs, subject="biology")
+process_business = partial(process_docs, subject="business")
+process_chemistry = partial(process_docs, subject="chemistry")
+process_computer_science = partial(process_docs, subject="computer science")
+process_economics = partial(process_docs, subject="economics")
+process_engineering = partial(process_docs, subject="engineering")
+process_health = partial(process_docs, subject="health")
+process_history = partial(process_docs, subject="history")
+process_law = partial(process_docs, subject="law")
+process_math = partial(process_docs, subject="math")
+process_other = partial(process_docs, subject="other")
+process_philosophy = partial(process_docs, subject="philosophy")
+process_physics = partial(process_docs, subject="physics")
+process_psychology = partial(process_docs, subject="psychology")

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "jurisprudence"
+"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\
+  \n"
+"tag": "mmlu_flan_n_shot_loglikelihood_humanities"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_jurisprudence"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "machine_learning"
+"description": "The following are multiple choice questions (with answers) about machine\
+  \ learning.\n\n"
+"tag": "mmlu_flan_n_shot_loglikelihood_stem"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_machine_learning"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "moral_scenarios"
+"description": "The following are multiple choice questions (with answers) about moral\
+  \ scenarios.\n\n"
+"tag": "mmlu_flan_n_shot_loglikelihood_humanities"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_moral_scenarios"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "philosophy"
+"description": "The following are multiple choice questions (with answers) about philosophy.\n\
+  \n"
+"tag": "mmlu_flan_n_shot_loglikelihood_humanities"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_philosophy"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "professional_accounting"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ accounting.\n\n"
+"tag": "mmlu_flan_n_shot_loglikelihood_other"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_professional_accounting"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "professional_law"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ law.\n\n"
+"tag": "mmlu_flan_n_shot_loglikelihood_humanities"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_professional_law"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "professional_medicine"
+"description": "The following are multiple choice questions (with answers) about professional\
+  \ medicine.\n\n"
+"tag": "mmlu_flan_n_shot_loglikelihood_other"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_professional_medicine"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "public_relations"
+"description": "The following are multiple choice questions (with answers) about public\
+  \ relations.\n\n"
+"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_public_relations"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "sociology"
+"description": "The following are multiple choice questions (with answers) about sociology.\n\
+  \n"
+"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_sociology"

lm-evaluation-harness/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml

@@ -0,0 +1,6 @@
+"dataset_name": "virology"
+"description": "The following are multiple choice questions (with answers) about virology.\n\
+  \n"
+"tag": "mmlu_flan_n_shot_loglikelihood_other"
+"include": "_mmlu_flan_loglikelihood_template_yaml"
+"task": "mmlu_flan_n_shot_loglikelihood_virology"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "astronomy"
+"description": "The following are multiple choice questions (with answers) about astronomy.\n\
+  \n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_astronomy_generative"
+"task_alias": "astronomy"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_chemistry"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ chemistry.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_college_chemistry_generative"
+"task_alias": "college_chemistry"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_mathematics"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ mathematics.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_college_mathematics_generative"
+"task_alias": "college_mathematics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "college_medicine"
+"description": "The following are multiple choice questions (with answers) about college\
+  \ medicine.\n\n"
+"tag": "mmlu_other_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_college_medicine_generative"
+"task_alias": "college_medicine"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "computer_security"
+"description": "The following are multiple choice questions (with answers) about computer\
+  \ security.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_computer_security_generative"
+"task_alias": "computer_security"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "conceptual_physics"
+"description": "The following are multiple choice questions (with answers) about conceptual\
+  \ physics.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_conceptual_physics_generative"
+"task_alias": "conceptual_physics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "econometrics"
+"description": "The following are multiple choice questions (with answers) about econometrics.\n\
+  \n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_econometrics_generative"
+"task_alias": "econometrics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "electrical_engineering"
+"description": "The following are multiple choice questions (with answers) about electrical\
+  \ engineering.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_electrical_engineering_generative"
+"task_alias": "electrical_engineering"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "elementary_mathematics"
+"description": "The following are multiple choice questions (with answers) about elementary\
+  \ mathematics.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_elementary_mathematics_generative"
+"task_alias": "elementary_mathematics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "formal_logic"
+"description": "The following are multiple choice questions (with answers) about formal\
+  \ logic.\n\n"
+"tag": "mmlu_humanities_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_formal_logic_generative"
+"task_alias": "formal_logic"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "global_facts"
+"description": "The following are multiple choice questions (with answers) about global\
+  \ facts.\n\n"
+"tag": "mmlu_other_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_global_facts_generative"
+"task_alias": "global_facts"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_biology"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school biology.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_biology_generative"
+"task_alias": "high_school_biology"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_computer_science"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school computer science.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_computer_science_generative"
+"task_alias": "high_school_computer_science"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_european_history"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school european history.\n\n"
+"tag": "mmlu_humanities_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_european_history_generative"
+"task_alias": "high_school_european_history"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_geography"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school geography.\n\n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_geography_generative"
+"task_alias": "high_school_geography"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_government_and_politics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school government and politics.\n\n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_government_and_politics_generative"
+"task_alias": "high_school_government_and_politics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_macroeconomics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school macroeconomics.\n\n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_macroeconomics_generative"
+"task_alias": "high_school_macroeconomics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_mathematics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school mathematics.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_mathematics_generative"
+"task_alias": "high_school_mathematics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_microeconomics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school microeconomics.\n\n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_microeconomics_generative"
+"task_alias": "high_school_microeconomics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_physics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school physics.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_physics_generative"
+"task_alias": "high_school_physics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_psychology"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school psychology.\n\n"
+"tag": "mmlu_social_sciences_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_psychology_generative"
+"task_alias": "high_school_psychology"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_statistics"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school statistics.\n\n"
+"tag": "mmlu_stem_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_statistics_generative"
+"task_alias": "high_school_statistics"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_us_history"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school us history.\n\n"
+"tag": "mmlu_humanities_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_us_history_generative"
+"task_alias": "high_school_us_history"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "high_school_world_history"
+"description": "The following are multiple choice questions (with answers) about high\
+  \ school world history.\n\n"
+"tag": "mmlu_humanities_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_high_school_world_history_generative"
+"task_alias": "high_school_world_history"

lm-evaluation-harness/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml

@@ -0,0 +1,7 @@
+"dataset_name": "human_aging"
+"description": "The following are multiple choice questions (with answers) about human\
+  \ aging.\n\n"
+"tag": "mmlu_other_generative"
+"include": "_default_template_yaml"
+"task": "mmlu_human_aging_generative"
+"task_alias": "human_aging"